2024-11-18 20:40:34,598 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-18 20:40:34,610 main DEBUG Took 0.010654 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 20:40:34,611 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 20:40:34,611 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 20:40:34,613 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 20:40:34,614 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:40:34,625 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 20:40:34,640 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:40:34,641 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:40:34,642 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:40:34,642 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:40:34,643 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:40:34,644 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:40:34,645 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:40:34,645 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:40:34,646 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:40:34,647 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:40:34,647 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:40:34,648 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:40:34,649 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:40:34,649 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:40:34,650 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:40:34,650 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:40:34,651 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:40:34,651 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:40:34,652 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:40:34,652 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:40:34,653 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:40:34,653 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:40:34,654 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:40:34,654 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:40:34,655 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:40:34,655 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 20:40:34,657 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:40:34,659 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 20:40:34,662 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 20:40:34,662 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 20:40:34,664 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 20:40:34,665 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 20:40:34,675 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 20:40:34,679 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 20:40:34,681 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 20:40:34,682 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 20:40:34,683 main DEBUG createAppenders(={Console}) 2024-11-18 20:40:34,684 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-18 20:40:34,685 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-18 20:40:34,685 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-18 20:40:34,686 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 20:40:34,686 main DEBUG OutputStream closed 2024-11-18 20:40:34,687 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 20:40:34,687 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 20:40:34,687 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-18 20:40:34,791 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 20:40:34,794 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 20:40:34,796 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 20:40:34,798 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 20:40:34,799 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 20:40:34,799 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 20:40:34,800 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 20:40:34,800 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 20:40:34,801 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 20:40:34,801 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 20:40:34,802 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 20:40:34,802 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 20:40:34,803 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 20:40:34,803 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 20:40:34,804 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 20:40:34,804 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 20:40:34,805 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 20:40:34,806 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 20:40:34,809 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 20:40:34,810 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-18 20:40:34,810 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 20:40:34,811 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-18T20:40:35,211 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a 2024-11-18 20:40:35,215 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 20:40:35,216 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T20:40:35,227 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-18T20:40:35,255 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-18T20:40:35,277 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/cluster_b02ea204-637b-6699-1ed5-681f735b812e, deleteOnExit=true 2024-11-18T20:40:35,277 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-18T20:40:35,279 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/test.cache.data in system properties and HBase conf 2024-11-18T20:40:35,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/hadoop.tmp.dir in system properties and HBase conf 2024-11-18T20:40:35,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/hadoop.log.dir in system properties and HBase conf 2024-11-18T20:40:35,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-18T20:40:35,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-18T20:40:35,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-18T20:40:35,418 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T20:40:35,564 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T20:40:35,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-18T20:40:35,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-18T20:40:35,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-18T20:40:35,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T20:40:35,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-18T20:40:35,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-18T20:40:35,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T20:40:35,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T20:40:35,590 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-18T20:40:35,591 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/nfs.dump.dir in system properties and HBase conf 2024-11-18T20:40:35,591 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/java.io.tmpdir in system properties and HBase conf 2024-11-18T20:40:35,592 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T20:40:35,592 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-18T20:40:35,593 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-18T20:40:36,604 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-18T20:40:36,695 INFO [Time-limited test {}] log.Log(170): Logging initialized @2980ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-18T20:40:36,787 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T20:40:36,860 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T20:40:36,885 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T20:40:36,885 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T20:40:36,887 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-18T20:40:36,906 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T20:40:36,911 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/hadoop.log.dir/,AVAILABLE} 2024-11-18T20:40:36,912 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T20:40:37,140 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/java.io.tmpdir/jetty-localhost-42483-hadoop-hdfs-3_4_1-tests_jar-_-any-440194947600323484/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T20:40:37,147 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:42483} 2024-11-18T20:40:37,147 INFO [Time-limited test {}] server.Server(415): Started @3432ms 2024-11-18T20:40:37,562 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T20:40:37,569 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T20:40:37,570 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T20:40:37,571 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T20:40:37,571 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T20:40:37,572 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/hadoop.log.dir/,AVAILABLE} 2024-11-18T20:40:37,572 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T20:40:37,690 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@292d50c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/java.io.tmpdir/jetty-localhost-34621-hadoop-hdfs-3_4_1-tests_jar-_-any-17387363002264401636/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T20:40:37,691 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:34621} 2024-11-18T20:40:37,691 INFO [Time-limited test {}] server.Server(415): Started @3976ms 2024-11-18T20:40:37,760 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T20:40:38,228 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/cluster_b02ea204-637b-6699-1ed5-681f735b812e/data/data1/current/BP-576781707-172.17.0.2-1731962436354/current, will proceed with Du for space computation calculation, 2024-11-18T20:40:38,229 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/cluster_b02ea204-637b-6699-1ed5-681f735b812e/data/data2/current/BP-576781707-172.17.0.2-1731962436354/current, will proceed with Du for space computation calculation, 2024-11-18T20:40:38,291 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T20:40:38,367 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xefcb76f8f3f00e2d with lease ID 0xfc60855e9a19cc2a: Processing first storage report for DS-8a8df1f1-18f7-4514-9c72-4d0df7bbd2a8 from datanode DatanodeRegistration(127.0.0.1:41353, datanodeUuid=6d06baad-4844-40b0-a3b2-9cc76010c5af, infoPort=41483, infoSecurePort=0, ipcPort=43203, storageInfo=lv=-57;cid=testClusterID;nsid=1989562623;c=1731962436354) 2024-11-18T20:40:38,369 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xefcb76f8f3f00e2d with lease ID 0xfc60855e9a19cc2a: from storage DS-8a8df1f1-18f7-4514-9c72-4d0df7bbd2a8 node DatanodeRegistration(127.0.0.1:41353, datanodeUuid=6d06baad-4844-40b0-a3b2-9cc76010c5af, infoPort=41483, infoSecurePort=0, ipcPort=43203, storageInfo=lv=-57;cid=testClusterID;nsid=1989562623;c=1731962436354), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-18T20:40:38,370 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xefcb76f8f3f00e2d with lease ID 0xfc60855e9a19cc2a: Processing first storage report for DS-dff92348-886d-4806-8b63-84a12ea9d65a from datanode DatanodeRegistration(127.0.0.1:41353, datanodeUuid=6d06baad-4844-40b0-a3b2-9cc76010c5af, infoPort=41483, infoSecurePort=0, ipcPort=43203, storageInfo=lv=-57;cid=testClusterID;nsid=1989562623;c=1731962436354) 2024-11-18T20:40:38,370 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xefcb76f8f3f00e2d with lease ID 0xfc60855e9a19cc2a: from storage DS-dff92348-886d-4806-8b63-84a12ea9d65a node DatanodeRegistration(127.0.0.1:41353, datanodeUuid=6d06baad-4844-40b0-a3b2-9cc76010c5af, infoPort=41483, infoSecurePort=0, ipcPort=43203, storageInfo=lv=-57;cid=testClusterID;nsid=1989562623;c=1731962436354), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T20:40:38,403 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a 2024-11-18T20:40:38,505 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/cluster_b02ea204-637b-6699-1ed5-681f735b812e/zookeeper_0, clientPort=56641, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/cluster_b02ea204-637b-6699-1ed5-681f735b812e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/cluster_b02ea204-637b-6699-1ed5-681f735b812e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T20:40:38,516 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56641 2024-11-18T20:40:38,531 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:40:38,537 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:40:38,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741825_1001 (size=7) 2024-11-18T20:40:39,336 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093 with version=8 2024-11-18T20:40:39,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/hbase-staging 2024-11-18T20:40:39,437 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T20:40:39,635 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c0a89b2656d4:0 server-side Connection retries=45 2024-11-18T20:40:39,644 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T20:40:39,644 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T20:40:39,648 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T20:40:39,648 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T20:40:39,649 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T20:40:39,771 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T20:40:39,830 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T20:40:39,843 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T20:40:39,848 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T20:40:39,879 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 121625 (auto-detected) 2024-11-18T20:40:39,880 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-18T20:40:39,899 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46825 2024-11-18T20:40:39,920 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46825 connecting to ZooKeeper ensemble=127.0.0.1:56641 2024-11-18T20:40:39,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:468250x0, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T20:40:39,953 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46825-0x1005490dfb30000 connected 2024-11-18T20:40:39,981 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:40:39,984 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:40:39,995 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T20:40:39,998 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093, hbase.cluster.distributed=false 2024-11-18T20:40:40,026 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T20:40:40,031 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46825 2024-11-18T20:40:40,031 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46825 2024-11-18T20:40:40,033 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46825 2024-11-18T20:40:40,034 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46825 2024-11-18T20:40:40,038 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46825 2024-11-18T20:40:40,146 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c0a89b2656d4:0 server-side Connection retries=45 2024-11-18T20:40:40,148 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T20:40:40,149 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T20:40:40,149 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T20:40:40,149 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T20:40:40,149 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T20:40:40,152 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T20:40:40,156 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T20:40:40,157 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45393 2024-11-18T20:40:40,159 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45393 connecting to ZooKeeper ensemble=127.0.0.1:56641 2024-11-18T20:40:40,161 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:40:40,165 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:40:40,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:453930x0, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T20:40:40,176 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45393-0x1005490dfb30001 connected 2024-11-18T20:40:40,176 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:453930x0, quorum=127.0.0.1:56641, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T20:40:40,181 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T20:40:40,192 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T20:40:40,194 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T20:40:40,200 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T20:40:40,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45393 2024-11-18T20:40:40,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45393 2024-11-18T20:40:40,202 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45393 2024-11-18T20:40:40,202 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45393 2024-11-18T20:40:40,203 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45393 2024-11-18T20:40:40,224 DEBUG [M:0;c0a89b2656d4:46825 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c0a89b2656d4:46825 2024-11-18T20:40:40,225 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c0a89b2656d4,46825,1731962439484 2024-11-18T20:40:40,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T20:40:40,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T20:40:40,235 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c0a89b2656d4,46825,1731962439484 2024-11-18T20:40:40,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T20:40:40,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:40:40,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:40:40,267 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T20:40:40,268 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c0a89b2656d4,46825,1731962439484 from backup master directory 2024-11-18T20:40:40,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T20:40:40,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c0a89b2656d4,46825,1731962439484 2024-11-18T20:40:40,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T20:40:40,272 WARN [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T20:40:40,273 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c0a89b2656d4,46825,1731962439484 2024-11-18T20:40:40,276 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-18T20:40:40,278 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-18T20:40:40,339 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/hbase.id] with ID: d9c0a363-374e-42c9-896f-88dfa8b27e5e 2024-11-18T20:40:40,339 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/.tmp/hbase.id 2024-11-18T20:40:40,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741826_1002 (size=42) 2024-11-18T20:40:40,354 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/.tmp/hbase.id]:[hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/hbase.id] 2024-11-18T20:40:40,411 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:40:40,416 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T20:40:40,440 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-11-18T20:40:40,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:40:40,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:40:40,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741827_1003 (size=196) 2024-11-18T20:40:40,488 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:40:40,493 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T20:40:40,512 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:40,517 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T20:40:40,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741828_1004 (size=1189) 2024-11-18T20:40:40,976 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store 2024-11-18T20:40:41,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741829_1005 (size=34) 2024-11-18T20:40:41,407 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T20:40:41,412 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:40:41,413 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T20:40:41,414 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:40:41,414 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:40:41,416 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T20:40:41,416 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:40:41,417 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:40:41,419 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731962441413Disabling compacts and flushes for region at 1731962441413Disabling writes for close at 1731962441416 (+3 ms)Writing region close event to WAL at 1731962441416Closed at 1731962441416 2024-11-18T20:40:41,422 WARN [master/c0a89b2656d4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/.initializing 2024-11-18T20:40:41,423 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/WALs/c0a89b2656d4,46825,1731962439484 2024-11-18T20:40:41,435 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T20:40:41,451 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c0a89b2656d4%2C46825%2C1731962439484, suffix=, logDir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/WALs/c0a89b2656d4,46825,1731962439484, archiveDir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/oldWALs, maxLogs=10 2024-11-18T20:40:41,479 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/WALs/c0a89b2656d4,46825,1731962439484/c0a89b2656d4%2C46825%2C1731962439484.1731962441456, exclude list is [], retry=0 2024-11-18T20:40:41,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41353,DS-8a8df1f1-18f7-4514-9c72-4d0df7bbd2a8,DISK] 2024-11-18T20:40:41,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-18T20:40:41,561 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/WALs/c0a89b2656d4,46825,1731962439484/c0a89b2656d4%2C46825%2C1731962439484.1731962441456 2024-11-18T20:40:41,562 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41483:41483)] 2024-11-18T20:40:41,563 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:40:41,564 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:40:41,567 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:40:41,568 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:40:41,608 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:40:41,647 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T20:40:41,651 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:40:41,654 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:40:41,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:40:41,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T20:40:41,660 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:40:41,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:40:41,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:40:41,666 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T20:40:41,666 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:40:41,667 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:40:41,667 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:40:41,670 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T20:40:41,671 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:40:41,672 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:40:41,672 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:40:41,676 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:40:41,678 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:40:41,683 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:40:41,684 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:40:41,688 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T20:40:41,692 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:40:41,697 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:40:41,698 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62643032, jitterRate=-0.06654608249664307}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T20:40:41,708 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731962441580Initializing all the Stores at 1731962441583 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731962441584 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962441585 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962441585Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962441585Cleaning up temporary data from old regions at 1731962441684 (+99 ms)Region opened successfully at 1731962441707 (+23 ms) 2024-11-18T20:40:41,709 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T20:40:41,747 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c1bb4de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c0a89b2656d4/172.17.0.2:0 2024-11-18T20:40:41,776 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T20:40:41,785 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T20:40:41,785 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T20:40:41,788 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T20:40:41,789 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-18T20:40:41,793 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-18T20:40:41,793 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T20:40:41,831 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T20:40:41,843 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T20:40:41,844 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T20:40:41,847 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T20:40:41,849 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T20:40:41,851 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T20:40:41,854 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T20:40:41,859 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T20:40:41,860 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T20:40:41,863 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T20:40:41,864 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T20:40:41,882 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T20:40:41,883 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T20:40:41,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T20:40:41,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T20:40:41,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:40:41,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:40:41,894 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c0a89b2656d4,46825,1731962439484, sessionid=0x1005490dfb30000, setting cluster-up flag (Was=false) 2024-11-18T20:40:41,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:40:41,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:40:41,913 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T20:40:41,915 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c0a89b2656d4,46825,1731962439484 2024-11-18T20:40:41,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:40:41,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:40:41,927 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T20:40:41,929 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c0a89b2656d4,46825,1731962439484 2024-11-18T20:40:41,936 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T20:40:42,010 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(746): ClusterId : d9c0a363-374e-42c9-896f-88dfa8b27e5e 2024-11-18T20:40:42,013 DEBUG [RS:0;c0a89b2656d4:45393 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T20:40:42,019 DEBUG [RS:0;c0a89b2656d4:45393 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T20:40:42,021 DEBUG [RS:0;c0a89b2656d4:45393 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T20:40:42,024 DEBUG [RS:0;c0a89b2656d4:45393 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T20:40:42,025 DEBUG [RS:0;c0a89b2656d4:45393 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48393c1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c0a89b2656d4/172.17.0.2:0 2024-11-18T20:40:42,025 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T20:40:42,037 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T20:40:42,044 DEBUG [RS:0;c0a89b2656d4:45393 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c0a89b2656d4:45393 2024-11-18T20:40:42,047 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T20:40:42,048 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T20:40:42,049 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T20:40:42,049 DEBUG [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T20:40:42,052 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(2659): reportForDuty to master=c0a89b2656d4,46825,1731962439484 with port=45393, startcode=1731962440107 2024-11-18T20:40:42,053 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c0a89b2656d4,46825,1731962439484 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T20:40:42,061 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c0a89b2656d4:0, corePoolSize=5, maxPoolSize=5 2024-11-18T20:40:42,061 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c0a89b2656d4:0, corePoolSize=5, maxPoolSize=5 2024-11-18T20:40:42,061 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c0a89b2656d4:0, corePoolSize=5, maxPoolSize=5 2024-11-18T20:40:42,062 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c0a89b2656d4:0, corePoolSize=5, maxPoolSize=5 2024-11-18T20:40:42,062 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c0a89b2656d4:0, corePoolSize=10, maxPoolSize=10 2024-11-18T20:40:42,062 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:40:42,062 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c0a89b2656d4:0, corePoolSize=2, maxPoolSize=2 2024-11-18T20:40:42,063 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:40:42,067 DEBUG [RS:0;c0a89b2656d4:45393 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T20:40:42,072 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731962472072 2024-11-18T20:40:42,073 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T20:40:42,074 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T20:40:42,074 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T20:40:42,075 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T20:40:42,078 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T20:40:42,080 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:40:42,080 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T20:40:42,081 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T20:40:42,081 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T20:40:42,082 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T20:40:42,084 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:42,090 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T20:40:42,091 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T20:40:42,092 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T20:40:42,099 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T20:40:42,100 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T20:40:42,105 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c0a89b2656d4:0:becomeActiveMaster-HFileCleaner.large.0-1731962442101,5,FailOnTimeoutGroup] 2024-11-18T20:40:42,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741831_1007 (size=1321) 2024-11-18T20:40:42,106 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c0a89b2656d4:0:becomeActiveMaster-HFileCleaner.small.0-1731962442105,5,FailOnTimeoutGroup] 2024-11-18T20:40:42,106 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:42,106 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T20:40:42,108 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:42,108 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:42,109 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53137, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T20:40:42,115 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46825 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:42,117 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46825 {}] master.ServerManager(517): Registering regionserver=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:42,136 DEBUG [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093 2024-11-18T20:40:42,136 DEBUG [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46353 2024-11-18T20:40:42,136 DEBUG [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T20:40:42,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T20:40:42,141 DEBUG [RS:0;c0a89b2656d4:45393 {}] zookeeper.ZKUtil(111): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:42,142 WARN [RS:0;c0a89b2656d4:45393 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T20:40:42,142 INFO [RS:0;c0a89b2656d4:45393 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T20:40:42,142 DEBUG [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/WALs/c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:42,147 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c0a89b2656d4,45393,1731962440107] 2024-11-18T20:40:42,173 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T20:40:42,193 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T20:40:42,199 INFO [RS:0;c0a89b2656d4:45393 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T20:40:42,199 INFO [RS:0;c0a89b2656d4:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:42,201 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T20:40:42,208 INFO [RS:0;c0a89b2656d4:45393 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T20:40:42,210 INFO [RS:0;c0a89b2656d4:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:42,211 DEBUG [RS:0;c0a89b2656d4:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:40:42,211 DEBUG [RS:0;c0a89b2656d4:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:40:42,211 DEBUG [RS:0;c0a89b2656d4:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:40:42,212 DEBUG [RS:0;c0a89b2656d4:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:40:42,212 DEBUG [RS:0;c0a89b2656d4:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:40:42,212 DEBUG [RS:0;c0a89b2656d4:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c0a89b2656d4:0, corePoolSize=2, maxPoolSize=2 2024-11-18T20:40:42,212 DEBUG [RS:0;c0a89b2656d4:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:40:42,212 DEBUG [RS:0;c0a89b2656d4:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:40:42,213 DEBUG [RS:0;c0a89b2656d4:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:40:42,213 DEBUG [RS:0;c0a89b2656d4:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:40:42,213 DEBUG [RS:0;c0a89b2656d4:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:40:42,213 DEBUG [RS:0;c0a89b2656d4:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:40:42,213 DEBUG [RS:0;c0a89b2656d4:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c0a89b2656d4:0, corePoolSize=3, maxPoolSize=3 2024-11-18T20:40:42,214 DEBUG [RS:0;c0a89b2656d4:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0, corePoolSize=3, maxPoolSize=3 2024-11-18T20:40:42,215 INFO [RS:0;c0a89b2656d4:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:42,215 INFO [RS:0;c0a89b2656d4:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:42,215 INFO [RS:0;c0a89b2656d4:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:42,215 INFO [RS:0;c0a89b2656d4:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:42,215 INFO [RS:0;c0a89b2656d4:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:42,215 INFO [RS:0;c0a89b2656d4:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=c0a89b2656d4,45393,1731962440107-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T20:40:42,232 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T20:40:42,234 INFO [RS:0;c0a89b2656d4:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=c0a89b2656d4,45393,1731962440107-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:42,234 INFO [RS:0;c0a89b2656d4:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:42,234 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.Replication(171): c0a89b2656d4,45393,1731962440107 started 2024-11-18T20:40:42,251 INFO [RS:0;c0a89b2656d4:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:42,251 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(1482): Serving as c0a89b2656d4,45393,1731962440107, RpcServer on c0a89b2656d4/172.17.0.2:45393, sessionid=0x1005490dfb30001 2024-11-18T20:40:42,252 DEBUG [RS:0;c0a89b2656d4:45393 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T20:40:42,252 DEBUG [RS:0;c0a89b2656d4:45393 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:42,252 DEBUG [RS:0;c0a89b2656d4:45393 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c0a89b2656d4,45393,1731962440107' 2024-11-18T20:40:42,252 DEBUG [RS:0;c0a89b2656d4:45393 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T20:40:42,253 DEBUG [RS:0;c0a89b2656d4:45393 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T20:40:42,254 DEBUG [RS:0;c0a89b2656d4:45393 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T20:40:42,254 DEBUG [RS:0;c0a89b2656d4:45393 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T20:40:42,254 DEBUG [RS:0;c0a89b2656d4:45393 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:42,254 DEBUG [RS:0;c0a89b2656d4:45393 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c0a89b2656d4,45393,1731962440107' 2024-11-18T20:40:42,254 DEBUG [RS:0;c0a89b2656d4:45393 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T20:40:42,255 DEBUG [RS:0;c0a89b2656d4:45393 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T20:40:42,256 DEBUG [RS:0;c0a89b2656d4:45393 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T20:40:42,256 INFO [RS:0;c0a89b2656d4:45393 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T20:40:42,256 INFO [RS:0;c0a89b2656d4:45393 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T20:40:42,363 INFO [RS:0;c0a89b2656d4:45393 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T20:40:42,367 INFO [RS:0;c0a89b2656d4:45393 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c0a89b2656d4%2C45393%2C1731962440107, suffix=, logDir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/WALs/c0a89b2656d4,45393,1731962440107, archiveDir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/oldWALs, maxLogs=32 2024-11-18T20:40:42,384 DEBUG [RS:0;c0a89b2656d4:45393 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/WALs/c0a89b2656d4,45393,1731962440107/c0a89b2656d4%2C45393%2C1731962440107.1731962442369, exclude list is [], retry=0 2024-11-18T20:40:42,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41353,DS-8a8df1f1-18f7-4514-9c72-4d0df7bbd2a8,DISK] 2024-11-18T20:40:42,391 INFO [RS:0;c0a89b2656d4:45393 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/WALs/c0a89b2656d4,45393,1731962440107/c0a89b2656d4%2C45393%2C1731962440107.1731962442369 2024-11-18T20:40:42,392 DEBUG [RS:0;c0a89b2656d4:45393 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41483:41483)] 2024-11-18T20:40:42,510 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T20:40:42,511 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093 2024-11-18T20:40:42,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741833_1009 (size=32) 2024-11-18T20:40:42,930 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:40:42,935 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T20:40:42,939 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T20:40:42,939 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:40:42,940 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:40:42,941 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T20:40:42,944 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T20:40:42,944 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:40:42,945 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:40:42,945 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T20:40:42,947 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T20:40:42,947 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:40:42,948 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:40:42,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T20:40:42,951 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T20:40:42,951 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:40:42,952 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:40:42,952 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T20:40:42,954 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740 2024-11-18T20:40:42,955 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740 2024-11-18T20:40:42,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T20:40:42,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T20:40:42,959 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:40:42,961 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T20:40:42,965 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:40:42,966 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70718474, jitterRate=0.05378738045692444}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:40:42,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731962442930Initializing all the Stores at 1731962442935 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731962442935Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731962442935Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962442935Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731962442935Cleaning up temporary data from old regions at 1731962442958 (+23 ms)Region opened successfully at 1731962442968 (+10 ms) 2024-11-18T20:40:42,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T20:40:42,969 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T20:40:42,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T20:40:42,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T20:40:42,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T20:40:42,970 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T20:40:42,970 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731962442968Disabling compacts and flushes for region at 1731962442968Disabling writes for close at 1731962442969 (+1 ms)Writing region close event to WAL at 1731962442970 (+1 ms)Closed at 1731962442970 2024-11-18T20:40:42,973 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T20:40:42,973 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T20:40:42,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T20:40:42,986 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T20:40:42,989 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T20:40:43,146 DEBUG [c0a89b2656d4:46825 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-18T20:40:43,160 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:43,165 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c0a89b2656d4,45393,1731962440107, state=OPENING 2024-11-18T20:40:43,170 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T20:40:43,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:40:43,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:40:43,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T20:40:43,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T20:40:43,174 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T20:40:43,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:40:43,355 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T20:40:43,359 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40693, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T20:40:43,369 INFO [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T20:40:43,370 INFO [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T20:40:43,370 INFO [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T20:40:43,374 INFO [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c0a89b2656d4%2C45393%2C1731962440107.meta, suffix=.meta, logDir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/WALs/c0a89b2656d4,45393,1731962440107, archiveDir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/oldWALs, maxLogs=32 2024-11-18T20:40:43,388 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/WALs/c0a89b2656d4,45393,1731962440107/c0a89b2656d4%2C45393%2C1731962440107.meta.1731962443376.meta, exclude list is [], retry=0 2024-11-18T20:40:43,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41353,DS-8a8df1f1-18f7-4514-9c72-4d0df7bbd2a8,DISK] 2024-11-18T20:40:43,395 INFO [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/WALs/c0a89b2656d4,45393,1731962440107/c0a89b2656d4%2C45393%2C1731962440107.meta.1731962443376.meta 2024-11-18T20:40:43,395 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41483:41483)] 2024-11-18T20:40:43,396 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:40:43,398 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T20:40:43,400 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T20:40:43,405 INFO [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T20:40:43,408 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T20:40:43,409 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:40:43,409 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T20:40:43,410 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T20:40:43,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T20:40:43,414 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T20:40:43,414 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:40:43,415 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:40:43,415 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T20:40:43,417 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T20:40:43,417 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:40:43,418 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:40:43,418 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T20:40:43,419 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T20:40:43,419 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:40:43,420 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:40:43,420 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T20:40:43,422 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T20:40:43,422 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:40:43,422 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:40:43,423 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T20:40:43,425 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740 2024-11-18T20:40:43,427 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740 2024-11-18T20:40:43,430 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T20:40:43,430 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T20:40:43,431 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:40:43,434 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T20:40:43,436 INFO [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72105134, jitterRate=0.07445022463798523}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:40:43,436 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T20:40:43,439 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731962443410Writing region info on filesystem at 1731962443410Initializing all the Stores at 1731962443412 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731962443412Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731962443413 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962443413Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731962443413Cleaning up temporary data from old regions at 1731962443430 (+17 ms)Running coprocessor post-open hooks at 1731962443436 (+6 ms)Region opened successfully at 1731962443438 (+2 ms) 2024-11-18T20:40:43,445 INFO [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731962443344 2024-11-18T20:40:43,455 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T20:40:43,455 INFO [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T20:40:43,457 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:43,460 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c0a89b2656d4,45393,1731962440107, state=OPEN 2024-11-18T20:40:43,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T20:40:43,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T20:40:43,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T20:40:43,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T20:40:43,464 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:43,471 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T20:40:43,471 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c0a89b2656d4,45393,1731962440107 in 289 msec 2024-11-18T20:40:43,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T20:40:43,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 494 msec 2024-11-18T20:40:43,479 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T20:40:43,479 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T20:40:43,496 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:40:43,497 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:40:43,519 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:40:43,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56081, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:40:43,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5680 sec 2024-11-18T20:40:43,542 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731962443542, completionTime=-1 2024-11-18T20:40:43,544 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-18T20:40:43,545 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-18T20:40:43,567 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-18T20:40:43,567 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731962503567 2024-11-18T20:40:43,567 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731962563567 2024-11-18T20:40:43,567 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 22 msec 2024-11-18T20:40:43,569 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c0a89b2656d4,46825,1731962439484-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:43,569 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c0a89b2656d4,46825,1731962439484-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:43,570 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c0a89b2656d4,46825,1731962439484-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:43,571 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c0a89b2656d4:46825, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:43,571 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:43,572 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:43,578 DEBUG [master/c0a89b2656d4:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T20:40:43,598 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.325sec 2024-11-18T20:40:43,599 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T20:40:43,600 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T20:40:43,601 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T20:40:43,602 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T20:40:43,602 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T20:40:43,602 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c0a89b2656d4,46825,1731962439484-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T20:40:43,603 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c0a89b2656d4,46825,1731962439484-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T20:40:43,612 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T20:40:43,612 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T20:40:43,613 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c0a89b2656d4,46825,1731962439484-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T20:40:43,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8a5f59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:43,619 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T20:40:43,619 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T20:40:43,622 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:40:43,624 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:40:43,631 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:40:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:40:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:40:43,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42058ae4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:43,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:40:43,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:40:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:40:43,641 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:40:43,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@650c631a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:43,644 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:40:43,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:40:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:40:43,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:40:43,688 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c0a89b2656d4,46825,1731962439484 2024-11-18T20:40:43,703 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=277, ProcessCount=11, AvailableMemoryMB=3635 2024-11-18T20:40:43,711 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:40:43,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c0a89b2656d4,46825,1731962439484 2024-11-18T20:40:43,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1cd49ee5 2024-11-18T20:40:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:40:43,727 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:40:43,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:40:43,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:40:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T20:40:43,743 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T20:40:43,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-18T20:40:43,745 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:40:43,748 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T20:40:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T20:40:43,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741835_1011 (size=1061) 2024-11-18T20:40:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T20:40:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T20:40:44,169 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093 2024-11-18T20:40:44,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741836_1012 (size=53) 2024-11-18T20:40:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T20:40:44,582 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:40:44,582 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4420199d334b1e279fd666dd4cb607ce, disabling compactions & flushes 2024-11-18T20:40:44,582 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:44,583 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:44,583 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. after waiting 0 ms 2024-11-18T20:40:44,583 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:44,583 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:44,583 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4420199d334b1e279fd666dd4cb607ce: Waiting for close lock at 1731962444582Disabling compacts and flushes for region at 1731962444582Disabling writes for close at 1731962444583 (+1 ms)Writing region close event to WAL at 1731962444583Closed at 1731962444583 2024-11-18T20:40:44,589 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T20:40:44,596 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731962444589"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731962444589"}]},"ts":"1731962444589"} 2024-11-18T20:40:44,602 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T20:40:44,604 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T20:40:44,607 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962444604"}]},"ts":"1731962444604"} 2024-11-18T20:40:44,611 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T20:40:44,613 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4420199d334b1e279fd666dd4cb607ce, ASSIGN}] 2024-11-18T20:40:44,615 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4420199d334b1e279fd666dd4cb607ce, ASSIGN 2024-11-18T20:40:44,618 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4420199d334b1e279fd666dd4cb607ce, ASSIGN; state=OFFLINE, location=c0a89b2656d4,45393,1731962440107; forceNewPlan=false, retain=false 2024-11-18T20:40:44,770 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4420199d334b1e279fd666dd4cb607ce, regionState=OPENING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:44,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4420199d334b1e279fd666dd4cb607ce, ASSIGN because future has completed 2024-11-18T20:40:44,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:40:44,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T20:40:44,947 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:44,948 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:40:44,948 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:44,948 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:40:44,948 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:44,949 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:44,951 INFO [StoreOpener-4420199d334b1e279fd666dd4cb607ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:44,955 INFO [StoreOpener-4420199d334b1e279fd666dd4cb607ce-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:40:44,955 INFO [StoreOpener-4420199d334b1e279fd666dd4cb607ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4420199d334b1e279fd666dd4cb607ce columnFamilyName A 2024-11-18T20:40:44,955 DEBUG [StoreOpener-4420199d334b1e279fd666dd4cb607ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:40:44,956 INFO [StoreOpener-4420199d334b1e279fd666dd4cb607ce-1 {}] regionserver.HStore(327): Store=4420199d334b1e279fd666dd4cb607ce/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:40:44,957 INFO [StoreOpener-4420199d334b1e279fd666dd4cb607ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:44,959 INFO [StoreOpener-4420199d334b1e279fd666dd4cb607ce-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:40:44,959 INFO [StoreOpener-4420199d334b1e279fd666dd4cb607ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4420199d334b1e279fd666dd4cb607ce columnFamilyName B 2024-11-18T20:40:44,960 DEBUG [StoreOpener-4420199d334b1e279fd666dd4cb607ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:40:44,960 INFO [StoreOpener-4420199d334b1e279fd666dd4cb607ce-1 {}] regionserver.HStore(327): Store=4420199d334b1e279fd666dd4cb607ce/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:40:44,961 INFO [StoreOpener-4420199d334b1e279fd666dd4cb607ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:44,963 INFO [StoreOpener-4420199d334b1e279fd666dd4cb607ce-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:40:44,963 INFO [StoreOpener-4420199d334b1e279fd666dd4cb607ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4420199d334b1e279fd666dd4cb607ce columnFamilyName C 2024-11-18T20:40:44,963 DEBUG [StoreOpener-4420199d334b1e279fd666dd4cb607ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:40:44,964 INFO [StoreOpener-4420199d334b1e279fd666dd4cb607ce-1 {}] regionserver.HStore(327): Store=4420199d334b1e279fd666dd4cb607ce/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:40:44,964 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:44,964 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:44,966 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:44,967 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:44,968 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:44,969 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:44,969 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:40:44,972 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:44,975 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:40:44,976 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4420199d334b1e279fd666dd4cb607ce; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71593429, jitterRate=0.06682522594928741}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:40:44,976 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:44,977 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4420199d334b1e279fd666dd4cb607ce: Running coprocessor pre-open hook at 1731962444949Writing region info on filesystem at 1731962444949Initializing all the Stores at 1731962444951 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962444951Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962444951Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962444951Cleaning up temporary data from old regions at 1731962444969 (+18 ms)Running coprocessor post-open hooks at 1731962444976 (+7 ms)Region opened successfully at 1731962444977 (+1 ms) 2024-11-18T20:40:44,979 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., pid=6, masterSystemTime=1731962444934 2024-11-18T20:40:44,982 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:44,982 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:44,983 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4420199d334b1e279fd666dd4cb607ce, regionState=OPEN, openSeqNum=2, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:44,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 because future has completed 2024-11-18T20:40:44,993 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T20:40:44,993 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 in 210 msec 2024-11-18T20:40:44,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T20:40:44,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4420199d334b1e279fd666dd4cb607ce, ASSIGN in 380 msec 2024-11-18T20:40:45,000 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T20:40:45,000 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962445000"}]},"ts":"1731962445000"} 2024-11-18T20:40:45,004 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T20:40:45,006 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T20:40:45,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2710 sec 2024-11-18T20:40:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T20:40:45,900 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:40:45,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e4ddb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,907 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:40:45,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:40:45,909 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:40:45,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:40:45,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:40:45,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a04f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:40:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:40:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:40:45,912 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:40:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@435bed7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,914 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:40:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:40:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:40:45,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:40:45,920 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:40:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c0a89b2656d4,46825,1731962439484 2024-11-18T20:40:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6bf45604 2024-11-18T20:40:45,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:40:45,926 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:40:45,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@378e4d76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,931 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:40:45,931 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:40:45,933 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:40:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:40:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:40:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a9376a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:40:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:40:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:40:45,935 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:40:45,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4c56da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aae1f3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,937 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:40:45,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:40:45,938 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:40:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:40:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:40:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6b0c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:40:45,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:40:45,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:40:45,941 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:40:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cdb8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,943 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728f452, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,943 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:40:45,944 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:40:45,945 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:40:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:40:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:40:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:40:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:40:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:40:45,948 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:40:45,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@350afbaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,950 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:40:45,950 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:40:45,951 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:40:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:40:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:40:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:40:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:40:45,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:40:45,954 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:40:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253fe80a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76933ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,956 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:40:45,956 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:40:45,957 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:40:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:40:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:40:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fecaead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:40:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:40:45,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:40:45,960 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:40:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4652641e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32105378, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,963 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:40:45,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:40:45,964 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:40:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:40:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:40:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64ebea6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:40:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:40:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:40:45,967 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:40:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@638af609, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c4a9e35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,970 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:40:45,970 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:40:45,971 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:40:45,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:40:45,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:40:45,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e9550bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:40:45,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:40:45,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:40:45,973 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:40:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b38edab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d091df0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,975 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:40:45,976 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:40:45,977 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:40:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:40:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:40:45,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1020fb6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:40:45,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:40:45,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:40:45,980 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:40:45,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57865cab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@460c95ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,982 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:40:45,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:40:45,983 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:40:45,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:40:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:40:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@463070d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:40:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:40:45,985 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:40:45,986 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:40:45,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f5c6f22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:40:45,991 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:40:45,991 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:40:45,991 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:40:45,992 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:40:45,992 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:40:45,993 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:40:45,992 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:40:45,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:40:45,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:40:45,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:40:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:40:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:40:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:40:45,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:40:45,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:40:45,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:40:45,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:40:45,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:40:45,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:40:45,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:40:45,996 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:40:45,996 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:40:45,996 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:40:45,996 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:40:45,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:40:45,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:40:45,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:40:45,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:40:45,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:40:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:40:45,999 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:40:46,000 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:40:46,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:40:46,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:40:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:40:46,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:40:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:40:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:40:46,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:40:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-18T20:40:46,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:40:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:40:46,009 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:40:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:40:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:40:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:40:46,014 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:40:46,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:40:46,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:40:46,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:40:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:40:46,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T20:40:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:46,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:40:46,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:46,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:40:46,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:46,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:40:46,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:40:46,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:40:46,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:46,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:46,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:46,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:46,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962506181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962506191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/4af65aa116644db1b6b03be9846c609a is 50, key is test_row_0/A:col10/1731962446016/Put/seqid=0 2024-11-18T20:40:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962506191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:46,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962506195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962506206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741837_1013 (size=12001) 2024-11-18T20:40:46,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/4af65aa116644db1b6b03be9846c609a 2024-11-18T20:40:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:40:46,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962506358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962506359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962506359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962506360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962506360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/15b465e897ff406aad758581d2799675 is 50, key is test_row_0/B:col10/1731962446016/Put/seqid=0 2024-11-18T20:40:46,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:40:46,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741838_1014 (size=12001) 2024-11-18T20:40:46,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/15b465e897ff406aad758581d2799675 2024-11-18T20:40:46,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:46,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:46,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:46,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:46,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:46,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/30e522c075b44fd6a17380ec809dbcab is 50, key is test_row_0/C:col10/1731962446016/Put/seqid=0 2024-11-18T20:40:46,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741839_1015 (size=12001) 2024-11-18T20:40:46,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/30e522c075b44fd6a17380ec809dbcab 2024-11-18T20:40:46,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/4af65aa116644db1b6b03be9846c609a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/4af65aa116644db1b6b03be9846c609a 2024-11-18T20:40:46,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/4af65aa116644db1b6b03be9846c609a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-18T20:40:46,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/15b465e897ff406aad758581d2799675 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/15b465e897ff406aad758581d2799675 2024-11-18T20:40:46,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/15b465e897ff406aad758581d2799675, entries=150, sequenceid=13, filesize=11.7 K 2024-11-18T20:40:46,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/30e522c075b44fd6a17380ec809dbcab as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/30e522c075b44fd6a17380ec809dbcab 2024-11-18T20:40:46,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:40:46,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/30e522c075b44fd6a17380ec809dbcab, entries=150, sequenceid=13, filesize=11.7 K 2024-11-18T20:40:46,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:46,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4420199d334b1e279fd666dd4cb607ce in 511ms, sequenceid=13, compaction requested=false 2024-11-18T20:40:46,591 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T20:40:46,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:46,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:46,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:46,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:46,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:46,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:46,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:46,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=134.18 KB heapSize=399.89 KB 2024-11-18T20:40:46,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:40:46,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962446181, max timestamp=1731962446588Num uniques 859; 2024-11-18T20:40:46,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731962446181, max timestamp=1731962446588Num uniques 859; 2024-11-18T20:40:46,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:40:46,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T20:40:46,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:40:46,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:40:46,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:40:46,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:40:46,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:40:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:40:46,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/102dd0b2bb4b4bad8c8dbc4a1e2fdb47 is 50, key is test_row_0/A:col10/1731962446606/Put/seqid=0 2024-11-18T20:40:46,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741840_1016 (size=12001) 2024-11-18T20:40:46,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/102dd0b2bb4b4bad8c8dbc4a1e2fdb47 2024-11-18T20:40:46,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962506666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962506674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962506680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962506682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962506686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/139306673b1945f3ae8019906641aea4 is 50, key is test_row_0/B:col10/1731962446606/Put/seqid=0 2024-11-18T20:40:46,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741841_1017 (size=12001) 2024-11-18T20:40:46,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.26 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/139306673b1945f3ae8019906641aea4 2024-11-18T20:40:46,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:40:46,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:46,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:46,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:46,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:46,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:46,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/eade16475fbf4be39c2a09a734e1add5 is 50, key is test_row_0/C:col10/1731962446606/Put/seqid=0 2024-11-18T20:40:46,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962506788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741842_1018 (size=12001) 2024-11-18T20:40:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962506809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962506818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962506819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962506820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:46,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:40:46,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:46,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:46,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:46,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:46,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962507009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962507021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962507038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962507039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962507049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:40:47,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:47,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:47,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:47,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:40:47,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/eade16475fbf4be39c2a09a734e1add5 2024-11-18T20:40:47,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/102dd0b2bb4b4bad8c8dbc4a1e2fdb47 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/102dd0b2bb4b4bad8c8dbc4a1e2fdb47 2024-11-18T20:40:47,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:40:47,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:47,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:47,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:47,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/102dd0b2bb4b4bad8c8dbc4a1e2fdb47, entries=150, sequenceid=37, filesize=11.7 K 2024-11-18T20:40:47,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/139306673b1945f3ae8019906641aea4 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/139306673b1945f3ae8019906641aea4 2024-11-18T20:40:47,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/139306673b1945f3ae8019906641aea4, entries=150, sequenceid=37, filesize=11.7 K 2024-11-18T20:40:47,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/eade16475fbf4be39c2a09a734e1add5 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/eade16475fbf4be39c2a09a734e1add5 2024-11-18T20:40:47,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/eade16475fbf4be39c2a09a734e1add5, entries=150, sequenceid=37, filesize=11.7 K 2024-11-18T20:40:47,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~77.88 KB/79745, heapSize ~222.98 KB/228328, currentSize=107.34 KB/109920 for 4420199d334b1e279fd666dd4cb607ce in 731ms, sequenceid=37, compaction requested=false 2024-11-18T20:40:47,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:47,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T20:40:47,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:40:47,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:47,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:40:47,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:47,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:40:47,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:47,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/fe1591a6c1f0414ca3aa71d19edf30ca is 50, key is test_row_0/A:col10/1731962446675/Put/seqid=0 2024-11-18T20:40:47,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:40:47,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:47,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:47,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:47,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962507384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962507388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962507389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962507396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962507396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741843_1019 (size=14341) 2024-11-18T20:40:47,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962507509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962507509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962507518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962507519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962507519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:40:47,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:47,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:47,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:47,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:40:47,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:47,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:47,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:47,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962507718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962507719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962507739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962507739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962507739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:47,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:47,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/fe1591a6c1f0414ca3aa71d19edf30ca 2024-11-18T20:40:47,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:40:47,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:47,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:47,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:47,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/dd4f8f0ed60f40828ed99be4b163bedf is 50, key is test_row_0/B:col10/1731962446675/Put/seqid=0 2024-11-18T20:40:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:47,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741844_1020 (size=12001) 2024-11-18T20:40:47,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/dd4f8f0ed60f40828ed99be4b163bedf 2024-11-18T20:40:47,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/cb212756686d4cbc9073be3f668009fc is 50, key is test_row_0/C:col10/1731962446675/Put/seqid=0 2024-11-18T20:40:47,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741845_1021 (size=12001) 2024-11-18T20:40:47,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/cb212756686d4cbc9073be3f668009fc 2024-11-18T20:40:47,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/fe1591a6c1f0414ca3aa71d19edf30ca as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/fe1591a6c1f0414ca3aa71d19edf30ca 2024-11-18T20:40:48,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/fe1591a6c1f0414ca3aa71d19edf30ca, entries=200, sequenceid=57, filesize=14.0 K 2024-11-18T20:40:48,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/dd4f8f0ed60f40828ed99be4b163bedf as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/dd4f8f0ed60f40828ed99be4b163bedf 2024-11-18T20:40:48,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:40:48,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:48,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:48,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:48,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:48,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:48,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/dd4f8f0ed60f40828ed99be4b163bedf, entries=150, sequenceid=57, filesize=11.7 K 2024-11-18T20:40:48,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/cb212756686d4cbc9073be3f668009fc as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/cb212756686d4cbc9073be3f668009fc 2024-11-18T20:40:48,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962508028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:48,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:48,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962508029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:48,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/cb212756686d4cbc9073be3f668009fc, entries=150, sequenceid=57, filesize=11.7 K 2024-11-18T20:40:48,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4420199d334b1e279fd666dd4cb607ce in 700ms, sequenceid=57, compaction requested=true 2024-11-18T20:40:48,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:48,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:40:48,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:48,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:40:48,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:48,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:40:48,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:40:48,058 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:40:48,058 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:40:48,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:48,063 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:40:48,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T20:40:48,065 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:40:48,065 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/B is initiating minor compaction (all files) 2024-11-18T20:40:48,065 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/B in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:48,066 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/15b465e897ff406aad758581d2799675, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/139306673b1945f3ae8019906641aea4, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/dd4f8f0ed60f40828ed99be4b163bedf] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=35.2 K 2024-11-18T20:40:48,068 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/A is initiating minor compaction (all files) 2024-11-18T20:40:48,068 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/A in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:48,069 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/4af65aa116644db1b6b03be9846c609a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/102dd0b2bb4b4bad8c8dbc4a1e2fdb47, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/fe1591a6c1f0414ca3aa71d19edf30ca] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=37.4 K 2024-11-18T20:40:48,069 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 15b465e897ff406aad758581d2799675, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731962446016 2024-11-18T20:40:48,070 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 139306673b1945f3ae8019906641aea4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731962446189 2024-11-18T20:40:48,070 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4af65aa116644db1b6b03be9846c609a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731962446016 2024-11-18T20:40:48,072 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 102dd0b2bb4b4bad8c8dbc4a1e2fdb47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731962446189 2024-11-18T20:40:48,072 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting dd4f8f0ed60f40828ed99be4b163bedf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731962446675 2024-11-18T20:40:48,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:40:48,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:48,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:40:48,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:48,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:40:48,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:48,074 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe1591a6c1f0414ca3aa71d19edf30ca, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731962446673 2024-11-18T20:40:48,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/04545e466a0f4b2f90478a5c05574eb6 is 50, key is test_row_0/A:col10/1731962448066/Put/seqid=0 2024-11-18T20:40:48,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741846_1022 (size=19021) 2024-11-18T20:40:48,150 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#B#compaction#11 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:40:48,151 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#A#compaction#10 average throughput is 0.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:40:48,151 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/24234c65e79c4efdb0ff4ea97f97116c is 50, key is test_row_0/B:col10/1731962446675/Put/seqid=0 2024-11-18T20:40:48,152 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/e5cab452c6984fa585a3b094428b85f9 is 50, key is test_row_0/A:col10/1731962446675/Put/seqid=0 2024-11-18T20:40:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:40:48,166 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-18T20:40:48,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:40:48,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:48,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:48,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:48,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:48,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:48,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962508151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:48,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741847_1023 (size=12104) 2024-11-18T20:40:48,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741848_1024 (size=12104) 2024-11-18T20:40:48,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962508181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962508181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:48,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:48,214 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/e5cab452c6984fa585a3b094428b85f9 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/e5cab452c6984fa585a3b094428b85f9 2024-11-18T20:40:48,216 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/24234c65e79c4efdb0ff4ea97f97116c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/24234c65e79c4efdb0ff4ea97f97116c 2024-11-18T20:40:48,245 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/B of 4420199d334b1e279fd666dd4cb607ce into 24234c65e79c4efdb0ff4ea97f97116c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:40:48,245 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:48,246 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/A of 4420199d334b1e279fd666dd4cb607ce into e5cab452c6984fa585a3b094428b85f9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:40:48,247 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:48,249 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/A, priority=13, startTime=1731962448049; duration=0sec 2024-11-18T20:40:48,250 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:40:48,250 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:A 2024-11-18T20:40:48,250 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:40:48,251 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/B, priority=13, startTime=1731962448057; duration=0sec 2024-11-18T20:40:48,251 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:48,251 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:B 2024-11-18T20:40:48,253 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:40:48,253 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/C is initiating minor compaction (all files) 2024-11-18T20:40:48,254 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/C in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:48,254 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/30e522c075b44fd6a17380ec809dbcab, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/eade16475fbf4be39c2a09a734e1add5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/cb212756686d4cbc9073be3f668009fc] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=35.2 K 2024-11-18T20:40:48,255 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30e522c075b44fd6a17380ec809dbcab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731962446016 2024-11-18T20:40:48,256 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting eade16475fbf4be39c2a09a734e1add5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731962446189 2024-11-18T20:40:48,257 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb212756686d4cbc9073be3f668009fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731962446675 2024-11-18T20:40:48,284 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#C#compaction#12 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:40:48,285 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/506b721b57814260889287c0c96432cd is 50, key is test_row_0/C:col10/1731962446675/Put/seqid=0 2024-11-18T20:40:48,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962508288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:48,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741849_1025 (size=12104) 2024-11-18T20:40:48,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962508308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:48,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962508308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:48,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:48,331 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/506b721b57814260889287c0c96432cd as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/506b721b57814260889287c0c96432cd 2024-11-18T20:40:48,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:40:48,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:48,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:48,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:48,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:48,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:48,350 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/C of 4420199d334b1e279fd666dd4cb607ce into 506b721b57814260889287c0c96432cd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:40:48,351 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:48,351 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/C, priority=13, startTime=1731962448057; duration=0sec 2024-11-18T20:40:48,351 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:48,351 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:C 2024-11-18T20:40:48,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:40:48,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:48,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:48,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:48,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:48,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:48,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962508514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:48,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:48,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/04545e466a0f4b2f90478a5c05574eb6 2024-11-18T20:40:48,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962508538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:48,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:48,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962508538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:48,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:48,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962508547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:48,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962508548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:48,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/f2a1c83f0bed4334a3b978ab457575ac is 50, key is test_row_0/B:col10/1731962448066/Put/seqid=0 2024-11-18T20:40:48,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741850_1026 (size=12001) 2024-11-18T20:40:48,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/f2a1c83f0bed4334a3b978ab457575ac 2024-11-18T20:40:48,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:40:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:48,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:48,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/f431af3539334f0392db951e2b1110e8 is 50, key is test_row_0/C:col10/1731962448066/Put/seqid=0 2024-11-18T20:40:48,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741851_1027 (size=12001) 2024-11-18T20:40:48,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/f431af3539334f0392db951e2b1110e8 2024-11-18T20:40:48,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/04545e466a0f4b2f90478a5c05574eb6 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/04545e466a0f4b2f90478a5c05574eb6 2024-11-18T20:40:48,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/04545e466a0f4b2f90478a5c05574eb6, entries=300, sequenceid=74, filesize=18.6 K 2024-11-18T20:40:48,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/f2a1c83f0bed4334a3b978ab457575ac as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f2a1c83f0bed4334a3b978ab457575ac 2024-11-18T20:40:48,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f2a1c83f0bed4334a3b978ab457575ac, entries=150, sequenceid=74, filesize=11.7 K 2024-11-18T20:40:48,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/f431af3539334f0392db951e2b1110e8 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/f431af3539334f0392db951e2b1110e8 2024-11-18T20:40:48,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/f431af3539334f0392db951e2b1110e8, entries=150, sequenceid=74, filesize=11.7 K 2024-11-18T20:40:48,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 4420199d334b1e279fd666dd4cb607ce in 718ms, sequenceid=74, compaction requested=false 2024-11-18T20:40:48,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:48,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:40:48,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:48,810 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T20:40:48,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:40:48,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:48,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:40:48,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:48,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:40:48,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:48,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/b5a0cabd58b24c20a5087d4b55032258 is 50, key is test_row_0/A:col10/1731962448125/Put/seqid=0 2024-11-18T20:40:48,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:48,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:48,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741852_1028 (size=12001) 2024-11-18T20:40:48,884 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/b5a0cabd58b24c20a5087d4b55032258 2024-11-18T20:40:48,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/42af97de7487424da6f38da4569bd638 is 50, key is test_row_0/B:col10/1731962448125/Put/seqid=0 2024-11-18T20:40:48,921 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T20:40:48,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741853_1029 (size=12001) 2024-11-18T20:40:48,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962508967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:48,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:48,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962508971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:48,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962508977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:48,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:49,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962509087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:49,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:49,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962509087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:49,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962509098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:49,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:49,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962509308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:49,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:49,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962509308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:49,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:49,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:49,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962509317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:49,361 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/42af97de7487424da6f38da4569bd638 2024-11-18T20:40:49,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/7cfdbcc1f97d4a70adef0e12bb04e566 is 50, key is test_row_0/C:col10/1731962448125/Put/seqid=0 2024-11-18T20:40:49,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741854_1030 (size=12001) 2024-11-18T20:40:49,428 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/7cfdbcc1f97d4a70adef0e12bb04e566 2024-11-18T20:40:49,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/b5a0cabd58b24c20a5087d4b55032258 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/b5a0cabd58b24c20a5087d4b55032258 2024-11-18T20:40:49,461 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/b5a0cabd58b24c20a5087d4b55032258, entries=150, sequenceid=93, filesize=11.7 K 2024-11-18T20:40:49,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/42af97de7487424da6f38da4569bd638 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/42af97de7487424da6f38da4569bd638 2024-11-18T20:40:49,479 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/42af97de7487424da6f38da4569bd638, entries=150, sequenceid=93, filesize=11.7 K 2024-11-18T20:40:49,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/7cfdbcc1f97d4a70adef0e12bb04e566 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/7cfdbcc1f97d4a70adef0e12bb04e566 2024-11-18T20:40:49,492 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/7cfdbcc1f97d4a70adef0e12bb04e566, entries=150, sequenceid=93, filesize=11.7 K 2024-11-18T20:40:49,496 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4420199d334b1e279fd666dd4cb607ce in 685ms, sequenceid=93, compaction requested=true 2024-11-18T20:40:49,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:49,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:49,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-18T20:40:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-18T20:40:49,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-18T20:40:49,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4850 sec 2024-11-18T20:40:49,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.5100 sec 2024-11-18T20:40:49,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T20:40:49,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:40:49,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:49,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:40:49,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:49,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:40:49,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:49,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/0ce60dfbeb954e09a18f483b16ca945e is 50, key is test_row_0/A:col10/1731962449563/Put/seqid=0 2024-11-18T20:40:49,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741855_1031 (size=14341) 2024-11-18T20:40:49,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/0ce60dfbeb954e09a18f483b16ca945e 2024-11-18T20:40:49,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/f21c01d78cef4aebaa83fa1334e2b5dc is 50, key is test_row_0/B:col10/1731962449563/Put/seqid=0 2024-11-18T20:40:49,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962509631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962509633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:49,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962509638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:49,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962509638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:49,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962509643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:49,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741856_1032 (size=12001) 2024-11-18T20:40:49,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/f21c01d78cef4aebaa83fa1334e2b5dc 2024-11-18T20:40:49,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/185f1ba088c041d18cc26efa0c2f1e71 is 50, key is test_row_0/C:col10/1731962449563/Put/seqid=0 2024-11-18T20:40:49,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741857_1033 (size=12001) 2024-11-18T20:40:49,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/185f1ba088c041d18cc26efa0c2f1e71 2024-11-18T20:40:49,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962509758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:49,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962509758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:49,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:49,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/0ce60dfbeb954e09a18f483b16ca945e as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0ce60dfbeb954e09a18f483b16ca945e 2024-11-18T20:40:49,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962509768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:49,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0ce60dfbeb954e09a18f483b16ca945e, entries=200, sequenceid=110, filesize=14.0 K 2024-11-18T20:40:49,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962509777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:49,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/f21c01d78cef4aebaa83fa1334e2b5dc as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f21c01d78cef4aebaa83fa1334e2b5dc 2024-11-18T20:40:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:49,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f21c01d78cef4aebaa83fa1334e2b5dc, entries=150, sequenceid=110, filesize=11.7 K 2024-11-18T20:40:49,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/185f1ba088c041d18cc26efa0c2f1e71 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/185f1ba088c041d18cc26efa0c2f1e71 2024-11-18T20:40:49,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/185f1ba088c041d18cc26efa0c2f1e71, entries=150, sequenceid=110, filesize=11.7 K 2024-11-18T20:40:49,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4420199d334b1e279fd666dd4cb607ce in 249ms, sequenceid=110, compaction requested=true 2024-11-18T20:40:49,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:49,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:40:49,815 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:40:49,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:49,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:40:49,815 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:40:49,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:49,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:40:49,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:40:49,818 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:40:49,818 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57467 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:40:49,818 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/B is initiating minor compaction (all files) 2024-11-18T20:40:49,818 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/A is initiating minor compaction (all files) 2024-11-18T20:40:49,818 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/A in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:49,818 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/B in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:49,818 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/e5cab452c6984fa585a3b094428b85f9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/04545e466a0f4b2f90478a5c05574eb6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/b5a0cabd58b24c20a5087d4b55032258, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0ce60dfbeb954e09a18f483b16ca945e] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=56.1 K 2024-11-18T20:40:49,818 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/24234c65e79c4efdb0ff4ea97f97116c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f2a1c83f0bed4334a3b978ab457575ac, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/42af97de7487424da6f38da4569bd638, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f21c01d78cef4aebaa83fa1334e2b5dc] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=47.0 K 2024-11-18T20:40:49,819 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5cab452c6984fa585a3b094428b85f9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731962446675 2024-11-18T20:40:49,820 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 24234c65e79c4efdb0ff4ea97f97116c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731962446675 2024-11-18T20:40:49,821 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04545e466a0f4b2f90478a5c05574eb6, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731962447378 2024-11-18T20:40:49,821 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting f2a1c83f0bed4334a3b978ab457575ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731962448056 2024-11-18T20:40:49,822 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5a0cabd58b24c20a5087d4b55032258, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731962448125 2024-11-18T20:40:49,822 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 42af97de7487424da6f38da4569bd638, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731962448125 2024-11-18T20:40:49,822 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ce60dfbeb954e09a18f483b16ca945e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731962448851 2024-11-18T20:40:49,826 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting f21c01d78cef4aebaa83fa1334e2b5dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731962448851 2024-11-18T20:40:49,827 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-18T20:40:49,827 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-18T20:40:49,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T20:40:49,830 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-18T20:40:49,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-18T20:40:49,830 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-18T20:40:49,832 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T20:40:49,832 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-18T20:40:49,858 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#B#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:40:49,860 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#A#compaction#21 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:40:49,861 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/64b1d8b97a5042068c49fe5b53e68a9b is 50, key is test_row_0/B:col10/1731962449563/Put/seqid=0 2024-11-18T20:40:49,862 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/57a4cee4b162495a83c5e5ca9d0e307f is 50, key is test_row_0/A:col10/1731962449563/Put/seqid=0 2024-11-18T20:40:49,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741858_1034 (size=12241) 2024-11-18T20:40:49,907 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/57a4cee4b162495a83c5e5ca9d0e307f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/57a4cee4b162495a83c5e5ca9d0e307f 2024-11-18T20:40:49,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741859_1035 (size=12241) 2024-11-18T20:40:49,930 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/64b1d8b97a5042068c49fe5b53e68a9b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/64b1d8b97a5042068c49fe5b53e68a9b 2024-11-18T20:40:49,932 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/A of 4420199d334b1e279fd666dd4cb607ce into 57a4cee4b162495a83c5e5ca9d0e307f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:40:49,933 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:49,933 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/A, priority=12, startTime=1731962449815; duration=0sec 2024-11-18T20:40:49,936 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:40:49,936 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:A 2024-11-18T20:40:49,937 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:40:49,941 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:40:49,942 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/C is initiating minor compaction (all files) 2024-11-18T20:40:49,942 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/C in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:49,942 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/506b721b57814260889287c0c96432cd, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/f431af3539334f0392db951e2b1110e8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/7cfdbcc1f97d4a70adef0e12bb04e566, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/185f1ba088c041d18cc26efa0c2f1e71] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=47.0 K 2024-11-18T20:40:49,944 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 506b721b57814260889287c0c96432cd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731962446675 2024-11-18T20:40:49,945 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/B of 4420199d334b1e279fd666dd4cb607ce into 64b1d8b97a5042068c49fe5b53e68a9b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:40:49,945 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting f431af3539334f0392db951e2b1110e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731962448056 2024-11-18T20:40:49,945 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:49,945 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/B, priority=12, startTime=1731962449815; duration=0sec 2024-11-18T20:40:49,946 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:49,947 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:B 2024-11-18T20:40:49,947 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cfdbcc1f97d4a70adef0e12bb04e566, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731962448125 2024-11-18T20:40:49,948 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 185f1ba088c041d18cc26efa0c2f1e71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731962448851 2024-11-18T20:40:49,981 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#C#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:40:49,982 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/201d09dafac744cb84fc6e79a92c854b is 50, key is test_row_0/C:col10/1731962449563/Put/seqid=0 2024-11-18T20:40:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:49,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T20:40:49,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:40:49,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:49,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:40:49,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:49,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:40:49,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:50,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/0000e828360b46d3ac4d83b85c8cea0d is 50, key is test_row_0/A:col10/1731962449605/Put/seqid=0 2024-11-18T20:40:50,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741860_1036 (size=12241) 2024-11-18T20:40:50,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741861_1037 (size=12051) 2024-11-18T20:40:50,033 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/201d09dafac744cb84fc6e79a92c854b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/201d09dafac744cb84fc6e79a92c854b 2024-11-18T20:40:50,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962510022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/0000e828360b46d3ac4d83b85c8cea0d 2024-11-18T20:40:50,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962510024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962510033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962510034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,055 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/C of 4420199d334b1e279fd666dd4cb607ce into 201d09dafac744cb84fc6e79a92c854b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:40:50,055 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:50,055 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/C, priority=12, startTime=1731962449815; duration=0sec 2024-11-18T20:40:50,055 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:50,055 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:C 2024-11-18T20:40:50,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/a0909c66461e423b839dba3c6e2154ce is 50, key is test_row_0/B:col10/1731962449605/Put/seqid=0 2024-11-18T20:40:50,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741862_1038 (size=12051) 2024-11-18T20:40:50,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/a0909c66461e423b839dba3c6e2154ce 2024-11-18T20:40:50,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/5902745b6e7247f6a1c8e21da9bda59a is 50, key is test_row_0/C:col10/1731962449605/Put/seqid=0 2024-11-18T20:40:50,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741863_1039 (size=12051) 2024-11-18T20:40:50,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962510137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962510147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962510148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962510148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:40:50,171 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:40:50,171 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:40:50,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962510171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-18T20:40:50,177 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:40:50,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T20:40:50,180 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:40:50,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:40:50,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T20:40:50,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T20:40:50,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:50,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:50,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:50,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:50,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:50,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:50,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962510347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962510358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962510367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962510367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T20:40:50,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:50,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:50,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:50,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:50,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T20:40:50,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/5902745b6e7247f6a1c8e21da9bda59a 2024-11-18T20:40:50,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/0000e828360b46d3ac4d83b85c8cea0d as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0000e828360b46d3ac4d83b85c8cea0d 2024-11-18T20:40:50,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0000e828360b46d3ac4d83b85c8cea0d, entries=150, sequenceid=129, filesize=11.8 K 2024-11-18T20:40:50,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/a0909c66461e423b839dba3c6e2154ce as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a0909c66461e423b839dba3c6e2154ce 2024-11-18T20:40:50,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a0909c66461e423b839dba3c6e2154ce, entries=150, sequenceid=129, filesize=11.8 K 2024-11-18T20:40:50,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/5902745b6e7247f6a1c8e21da9bda59a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/5902745b6e7247f6a1c8e21da9bda59a 2024-11-18T20:40:50,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/5902745b6e7247f6a1c8e21da9bda59a, entries=150, sequenceid=129, filesize=11.8 K 2024-11-18T20:40:50,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 4420199d334b1e279fd666dd4cb607ce in 586ms, sequenceid=129, compaction requested=false 2024-11-18T20:40:50,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:50,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T20:40:50,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:50,648 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T20:40:50,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:40:50,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:50,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:40:50,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:50,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:40:50,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:50,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/daf3e7f7cff745e98f545f7f86d74cff is 50, key is test_row_0/A:col10/1731962450020/Put/seqid=0 2024-11-18T20:40:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:50,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:50,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741864_1040 (size=12151) 2024-11-18T20:40:50,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962510700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962510702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962510704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962510706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T20:40:50,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962510818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962510818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962510818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:50,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962510818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:50,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:50,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:51,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962511028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:51,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962511028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:51,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962511038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:51,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962511038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:51,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:51,088 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/daf3e7f7cff745e98f545f7f86d74cff 2024-11-18T20:40:51,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/e028f39b46b34b6887feeea0291446c9 is 50, key is test_row_0/B:col10/1731962450020/Put/seqid=0 2024-11-18T20:40:51,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741865_1041 (size=12151) 2024-11-18T20:40:51,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962511187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:51,192 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in 4420199d334b1e279fd666dd4cb607ce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:40:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T20:40:51,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962511337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962511337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:51,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962511347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:51,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962511348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:51,520 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/e028f39b46b34b6887feeea0291446c9 2024-11-18T20:40:51,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/cbf1d0d7e5f44c638959914076539d24 is 50, key is test_row_0/C:col10/1731962450020/Put/seqid=0 2024-11-18T20:40:51,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741866_1042 (size=12151) 2024-11-18T20:40:51,588 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/cbf1d0d7e5f44c638959914076539d24 2024-11-18T20:40:51,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/daf3e7f7cff745e98f545f7f86d74cff as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/daf3e7f7cff745e98f545f7f86d74cff 2024-11-18T20:40:51,607 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/daf3e7f7cff745e98f545f7f86d74cff, entries=150, sequenceid=146, filesize=11.9 K 2024-11-18T20:40:51,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/e028f39b46b34b6887feeea0291446c9 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/e028f39b46b34b6887feeea0291446c9 2024-11-18T20:40:51,618 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/e028f39b46b34b6887feeea0291446c9, entries=150, sequenceid=146, filesize=11.9 K 2024-11-18T20:40:51,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/cbf1d0d7e5f44c638959914076539d24 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/cbf1d0d7e5f44c638959914076539d24 2024-11-18T20:40:51,630 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/cbf1d0d7e5f44c638959914076539d24, entries=150, sequenceid=146, filesize=11.9 K 2024-11-18T20:40:51,631 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 4420199d334b1e279fd666dd4cb607ce in 983ms, sequenceid=146, compaction requested=true 2024-11-18T20:40:51,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:51,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:51,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-18T20:40:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-18T20:40:51,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-18T20:40:51,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4560 sec 2024-11-18T20:40:51,644 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.4690 sec 2024-11-18T20:40:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:51,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T20:40:51,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:40:51,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:51,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:40:51,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:51,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:40:51,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:51,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/0230ecc5f5a044f4959a40d8d03c67ec is 50, key is test_row_0/A:col10/1731962450701/Put/seqid=0 2024-11-18T20:40:51,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741867_1043 (size=12151) 2024-11-18T20:40:51,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962511879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:51,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:51,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962511882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:51,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:51,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962511882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:51,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962511885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:51,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:51,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:51,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962511987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:51,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:51,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962511998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:51,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:51,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:51,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962511998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:52,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:52,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962511998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:52,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:52,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962512197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:52,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:52,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962512208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:52,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:52,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962512209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:52,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962512209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:52,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:52,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/0230ecc5f5a044f4959a40d8d03c67ec 2024-11-18T20:40:52,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/131b4c37965243cdb3d0db1929558c60 is 50, key is test_row_0/B:col10/1731962450701/Put/seqid=0 2024-11-18T20:40:52,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741868_1044 (size=12151) 2024-11-18T20:40:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T20:40:52,328 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:40:52,328 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:40:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-18T20:40:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T20:40:52,331 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:40:52,332 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:40:52,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:40:52,336 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46825 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=c0a89b2656d4,45393,1731962440107, table=TestAcidGuarantees, region=4420199d334b1e279fd666dd4cb607ce. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-18T20:40:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T20:40:52,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:40:52,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:52,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:52,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:52,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:52,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:52,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962512507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:52,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:52,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962512518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962512518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962512518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:52,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:52,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:52,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:40:52,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:52,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:52,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:52,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:52,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T20:40:52,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/131b4c37965243cdb3d0db1929558c60 2024-11-18T20:40:52,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/db7d758a5d9540e8ab73b6b728783b16 is 50, key is test_row_0/C:col10/1731962450701/Put/seqid=0 2024-11-18T20:40:52,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741869_1045 (size=12151) 2024-11-18T20:40:52,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/db7d758a5d9540e8ab73b6b728783b16 2024-11-18T20:40:52,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/0230ecc5f5a044f4959a40d8d03c67ec as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0230ecc5f5a044f4959a40d8d03c67ec 2024-11-18T20:40:52,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0230ecc5f5a044f4959a40d8d03c67ec, entries=150, sequenceid=164, filesize=11.9 K 2024-11-18T20:40:52,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/131b4c37965243cdb3d0db1929558c60 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/131b4c37965243cdb3d0db1929558c60 2024-11-18T20:40:52,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/131b4c37965243cdb3d0db1929558c60, entries=150, sequenceid=164, filesize=11.9 K 2024-11-18T20:40:52,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/db7d758a5d9540e8ab73b6b728783b16 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/db7d758a5d9540e8ab73b6b728783b16 2024-11-18T20:40:52,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/db7d758a5d9540e8ab73b6b728783b16, entries=150, sequenceid=164, filesize=11.9 K 2024-11-18T20:40:52,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 4420199d334b1e279fd666dd4cb607ce in 941ms, sequenceid=164, compaction requested=true 2024-11-18T20:40:52,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:52,790 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:40:52,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:40:52,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:52,792 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:40:52,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:40:52,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:52,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:40:52,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:40:52,793 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:40:52,794 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/A is initiating minor compaction (all files) 2024-11-18T20:40:52,794 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/A in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:52,794 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/57a4cee4b162495a83c5e5ca9d0e307f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0000e828360b46d3ac4d83b85c8cea0d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/daf3e7f7cff745e98f545f7f86d74cff, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0230ecc5f5a044f4959a40d8d03c67ec] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=47.5 K 2024-11-18T20:40:52,795 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:40:52,795 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/B is initiating minor compaction (all files) 2024-11-18T20:40:52,795 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/B in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:52,795 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/64b1d8b97a5042068c49fe5b53e68a9b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a0909c66461e423b839dba3c6e2154ce, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/e028f39b46b34b6887feeea0291446c9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/131b4c37965243cdb3d0db1929558c60] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=47.5 K 2024-11-18T20:40:52,796 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57a4cee4b162495a83c5e5ca9d0e307f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731962448851 2024-11-18T20:40:52,797 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 64b1d8b97a5042068c49fe5b53e68a9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731962448851 2024-11-18T20:40:52,798 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting a0909c66461e423b839dba3c6e2154ce, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731962449605 2024-11-18T20:40:52,798 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0000e828360b46d3ac4d83b85c8cea0d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731962449605 2024-11-18T20:40:52,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:40:52,798 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting e028f39b46b34b6887feeea0291446c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731962450012 2024-11-18T20:40:52,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:52,799 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting daf3e7f7cff745e98f545f7f86d74cff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731962450012 2024-11-18T20:40:52,799 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T20:40:52,799 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 131b4c37965243cdb3d0db1929558c60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731962450701 2024-11-18T20:40:52,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:40:52,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:52,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:40:52,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:52,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:40:52,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:52,800 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0230ecc5f5a044f4959a40d8d03c67ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731962450701 2024-11-18T20:40:52,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/f7c9ccb5d82a4d33996f4df2d5b2b8c7 is 50, key is test_row_0/A:col10/1731962451881/Put/seqid=0 2024-11-18T20:40:52,828 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#B#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:40:52,829 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/a2d6d235e84c4973a1085683e3e40ee0 is 50, key is test_row_0/B:col10/1731962450701/Put/seqid=0 2024-11-18T20:40:52,834 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#A#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:40:52,835 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/85f5ff16c9ad4272a63dd45cbc124067 is 50, key is test_row_0/A:col10/1731962450701/Put/seqid=0 2024-11-18T20:40:52,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741870_1046 (size=12151) 2024-11-18T20:40:52,860 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/f7c9ccb5d82a4d33996f4df2d5b2b8c7 2024-11-18T20:40:52,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741872_1048 (size=12527) 2024-11-18T20:40:52,891 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/85f5ff16c9ad4272a63dd45cbc124067 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/85f5ff16c9ad4272a63dd45cbc124067 2024-11-18T20:40:52,903 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/A of 4420199d334b1e279fd666dd4cb607ce into 85f5ff16c9ad4272a63dd45cbc124067(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:40:52,903 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:52,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741871_1047 (size=12527) 2024-11-18T20:40:52,903 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/A, priority=12, startTime=1731962452790; duration=0sec 2024-11-18T20:40:52,904 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:40:52,904 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:A 2024-11-18T20:40:52,904 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:40:52,908 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:40:52,908 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/C is initiating minor compaction (all files) 2024-11-18T20:40:52,908 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/C in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:52,909 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/201d09dafac744cb84fc6e79a92c854b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/5902745b6e7247f6a1c8e21da9bda59a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/cbf1d0d7e5f44c638959914076539d24, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/db7d758a5d9540e8ab73b6b728783b16] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=47.5 K 2024-11-18T20:40:52,910 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 201d09dafac744cb84fc6e79a92c854b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731962448851 2024-11-18T20:40:52,910 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5902745b6e7247f6a1c8e21da9bda59a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731962449605 2024-11-18T20:40:52,911 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbf1d0d7e5f44c638959914076539d24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731962450012 2024-11-18T20:40:52,911 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting db7d758a5d9540e8ab73b6b728783b16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731962450701 2024-11-18T20:40:52,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/978f88640a6a463f82c77cc76a3d7f73 is 50, key is test_row_0/B:col10/1731962451881/Put/seqid=0 2024-11-18T20:40:52,917 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/a2d6d235e84c4973a1085683e3e40ee0 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a2d6d235e84c4973a1085683e3e40ee0 2024-11-18T20:40:52,927 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/B of 4420199d334b1e279fd666dd4cb607ce into a2d6d235e84c4973a1085683e3e40ee0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:40:52,927 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:52,927 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/B, priority=12, startTime=1731962452792; duration=0sec 2024-11-18T20:40:52,928 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:52,928 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:B 2024-11-18T20:40:52,936 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#C#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:40:52,937 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/d7705e12e9c74bb4953c0b951b6ac035 is 50, key is test_row_0/C:col10/1731962450701/Put/seqid=0 2024-11-18T20:40:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T20:40:52,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741873_1049 (size=12151) 2024-11-18T20:40:52,962 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/978f88640a6a463f82c77cc76a3d7f73 2024-11-18T20:40:52,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741874_1050 (size=12527) 2024-11-18T20:40:52,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/7c24bf3d6df64f7d85daf5a09c36bf08 is 50, key is test_row_0/C:col10/1731962451881/Put/seqid=0 2024-11-18T20:40:52,981 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/d7705e12e9c74bb4953c0b951b6ac035 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/d7705e12e9c74bb4953c0b951b6ac035 2024-11-18T20:40:52,997 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/C of 4420199d334b1e279fd666dd4cb607ce into d7705e12e9c74bb4953c0b951b6ac035(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:40:52,998 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:52,998 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/C, priority=12, startTime=1731962452792; duration=0sec 2024-11-18T20:40:52,998 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:52,998 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:C 2024-11-18T20:40:53,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:53,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:53,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741875_1051 (size=12151) 2024-11-18T20:40:53,038 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/7c24bf3d6df64f7d85daf5a09c36bf08 2024-11-18T20:40:53,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/f7c9ccb5d82a4d33996f4df2d5b2b8c7 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/f7c9ccb5d82a4d33996f4df2d5b2b8c7 2024-11-18T20:40:53,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962513051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962513053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962513054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962513056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,070 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/f7c9ccb5d82a4d33996f4df2d5b2b8c7, entries=150, sequenceid=179, filesize=11.9 K 2024-11-18T20:40:53,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/978f88640a6a463f82c77cc76a3d7f73 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/978f88640a6a463f82c77cc76a3d7f73 2024-11-18T20:40:53,082 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/978f88640a6a463f82c77cc76a3d7f73, entries=150, sequenceid=179, filesize=11.9 K 2024-11-18T20:40:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/7c24bf3d6df64f7d85daf5a09c36bf08 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/7c24bf3d6df64f7d85daf5a09c36bf08 2024-11-18T20:40:53,093 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/7c24bf3d6df64f7d85daf5a09c36bf08, entries=150, sequenceid=179, filesize=11.9 K 2024-11-18T20:40:53,094 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4420199d334b1e279fd666dd4cb607ce in 295ms, sequenceid=179, compaction requested=false 2024-11-18T20:40:53,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:53,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:53,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-18T20:40:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-18T20:40:53,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-18T20:40:53,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 765 msec 2024-11-18T20:40:53,108 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 776 msec 2024-11-18T20:40:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:53,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T20:40:53,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:40:53,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:53,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:40:53,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:53,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:40:53,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:53,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/30ac359f5f8146f49b9ab48daacec3e2 is 50, key is test_row_0/A:col10/1731962453054/Put/seqid=0 2024-11-18T20:40:53,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741876_1052 (size=16931) 2024-11-18T20:40:53,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962513186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962513189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962513191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962513193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962513207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,215 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in 4420199d334b1e279fd666dd4cb607ce of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:40:53,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962513297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962513307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962513307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962513308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T20:40:53,467 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:40:53,468 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:40:53,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-18T20:40:53,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T20:40:53,470 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:40:53,471 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:40:53,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:40:53,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962513508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962513517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962513517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962513517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T20:40:53,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/30ac359f5f8146f49b9ab48daacec3e2 2024-11-18T20:40:53,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/8cdbd4fb1c80457eb826d95968c666e3 is 50, key is test_row_0/B:col10/1731962453054/Put/seqid=0 2024-11-18T20:40:53,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741877_1053 (size=12151) 2024-11-18T20:40:53,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/8cdbd4fb1c80457eb826d95968c666e3 2024-11-18T20:40:53,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T20:40:53,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:53,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:53,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:53,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:53,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:53,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:53,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/e3159c52c8714038b7b10314ef262f4f is 50, key is test_row_0/C:col10/1731962453054/Put/seqid=0 2024-11-18T20:40:53,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741878_1054 (size=12151) 2024-11-18T20:40:53,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T20:40:53,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:53,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:53,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:53,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:53,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T20:40:53,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962513817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962513827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962513827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962513827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:53,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T20:40:53,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:53,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:53,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:53,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:53,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:54,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/e3159c52c8714038b7b10314ef262f4f 2024-11-18T20:40:54,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/30ac359f5f8146f49b9ab48daacec3e2 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/30ac359f5f8146f49b9ab48daacec3e2 2024-11-18T20:40:54,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/30ac359f5f8146f49b9ab48daacec3e2, entries=250, sequenceid=201, filesize=16.5 K 2024-11-18T20:40:54,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/8cdbd4fb1c80457eb826d95968c666e3 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/8cdbd4fb1c80457eb826d95968c666e3 2024-11-18T20:40:54,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T20:40:54,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:54,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:54,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:54,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:54,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:54,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/8cdbd4fb1c80457eb826d95968c666e3, entries=150, sequenceid=201, filesize=11.9 K 2024-11-18T20:40:54,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/e3159c52c8714038b7b10314ef262f4f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/e3159c52c8714038b7b10314ef262f4f 2024-11-18T20:40:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T20:40:54,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/e3159c52c8714038b7b10314ef262f4f, entries=150, sequenceid=201, filesize=11.9 K 2024-11-18T20:40:54,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4420199d334b1e279fd666dd4cb607ce in 937ms, sequenceid=201, compaction requested=true 2024-11-18T20:40:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:40:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:54,106 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:40:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:40:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:40:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:40:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:40:54,107 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:40:54,108 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:40:54,108 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41609 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:40:54,108 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/B is initiating minor compaction (all files) 2024-11-18T20:40:54,108 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/A is initiating minor compaction (all files) 2024-11-18T20:40:54,108 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/B in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:54,108 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/A in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:54,108 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/85f5ff16c9ad4272a63dd45cbc124067, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/f7c9ccb5d82a4d33996f4df2d5b2b8c7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/30ac359f5f8146f49b9ab48daacec3e2] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=40.6 K 2024-11-18T20:40:54,108 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a2d6d235e84c4973a1085683e3e40ee0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/978f88640a6a463f82c77cc76a3d7f73, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/8cdbd4fb1c80457eb826d95968c666e3] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=36.0 K 2024-11-18T20:40:54,109 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 85f5ff16c9ad4272a63dd45cbc124067, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731962450701 2024-11-18T20:40:54,109 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2d6d235e84c4973a1085683e3e40ee0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731962450701 2024-11-18T20:40:54,109 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 978f88640a6a463f82c77cc76a3d7f73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731962451862 2024-11-18T20:40:54,109 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting f7c9ccb5d82a4d33996f4df2d5b2b8c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731962451862 2024-11-18T20:40:54,110 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cdbd4fb1c80457eb826d95968c666e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731962453054 2024-11-18T20:40:54,110 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 30ac359f5f8146f49b9ab48daacec3e2, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731962453039 2024-11-18T20:40:54,124 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#B#compaction#42 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:40:54,125 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/f49ff82855434bf89c88b18d8e94c0a7 is 50, key is test_row_0/B:col10/1731962453054/Put/seqid=0 2024-11-18T20:40:54,136 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#A#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:40:54,136 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/263f3d31bc494878a3be61551a8c5f5a is 50, key is test_row_0/A:col10/1731962453054/Put/seqid=0 2024-11-18T20:40:54,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741879_1055 (size=12629) 2024-11-18T20:40:54,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741880_1056 (size=12629) 2024-11-18T20:40:54,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T20:40:54,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:54,242 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T20:40:54,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:40:54,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:54,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:40:54,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:54,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:40:54,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:54,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/497b14d533cb4a56bf8d100482542d0c is 50, key is test_row_0/A:col10/1731962453190/Put/seqid=0 2024-11-18T20:40:54,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741881_1057 (size=12151) 2024-11-18T20:40:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:54,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:54,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962514362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:54,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962514366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:54,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962514367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:54,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962514369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:54,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962514477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:54,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:54,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962514477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:54,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962514477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962514477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:54,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:54,566 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/f49ff82855434bf89c88b18d8e94c0a7 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f49ff82855434bf89c88b18d8e94c0a7 2024-11-18T20:40:54,575 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/263f3d31bc494878a3be61551a8c5f5a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/263f3d31bc494878a3be61551a8c5f5a 2024-11-18T20:40:54,576 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/B of 4420199d334b1e279fd666dd4cb607ce into f49ff82855434bf89c88b18d8e94c0a7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:40:54,576 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:54,576 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/B, priority=13, startTime=1731962454106; duration=0sec 2024-11-18T20:40:54,576 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:40:54,576 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:B 2024-11-18T20:40:54,577 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:40:54,579 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:40:54,580 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/C is initiating minor compaction (all files) 2024-11-18T20:40:54,580 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/C in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:54,580 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/d7705e12e9c74bb4953c0b951b6ac035, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/7c24bf3d6df64f7d85daf5a09c36bf08, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/e3159c52c8714038b7b10314ef262f4f] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=36.0 K 2024-11-18T20:40:54,581 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7705e12e9c74bb4953c0b951b6ac035, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731962450701 2024-11-18T20:40:54,581 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c24bf3d6df64f7d85daf5a09c36bf08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731962451862 2024-11-18T20:40:54,582 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3159c52c8714038b7b10314ef262f4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731962453054 2024-11-18T20:40:54,586 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/A of 4420199d334b1e279fd666dd4cb607ce into 263f3d31bc494878a3be61551a8c5f5a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:40:54,586 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:54,586 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/A, priority=13, startTime=1731962454106; duration=0sec 2024-11-18T20:40:54,588 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:54,589 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:A 2024-11-18T20:40:54,595 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:40:54,596 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/c5f698e6f0a9443da482b7c9c2c2ae52 is 50, key is test_row_0/C:col10/1731962453054/Put/seqid=0 2024-11-18T20:40:54,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741882_1058 (size=12629) 2024-11-18T20:40:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T20:40:54,653 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/497b14d533cb4a56bf8d100482542d0c 2024-11-18T20:40:54,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/571d2bd8172c4aa19f9371f050a4f9b7 is 50, key is test_row_0/B:col10/1731962453190/Put/seqid=0 2024-11-18T20:40:54,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741883_1059 (size=12151) 2024-11-18T20:40:54,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962514688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:54,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:54,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962514688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:54,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:54,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:54,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962514688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:54,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:54,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:54,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962514688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:54,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:54,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:55,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962514998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:55,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962514998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:55,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962514998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:55,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:55,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:55,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:55,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962514998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:55,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:55,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:55,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:55,014 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/c5f698e6f0a9443da482b7c9c2c2ae52 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/c5f698e6f0a9443da482b7c9c2c2ae52 2024-11-18T20:40:55,022 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/C of 4420199d334b1e279fd666dd4cb607ce into c5f698e6f0a9443da482b7c9c2c2ae52(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:40:55,022 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:55,022 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/C, priority=13, startTime=1731962454106; duration=0sec 2024-11-18T20:40:55,025 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:55,025 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:C 2024-11-18T20:40:55,078 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/571d2bd8172c4aa19f9371f050a4f9b7 2024-11-18T20:40:55,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/6f4a271719b5425ea874eb71e99e4a68 is 50, key is test_row_0/C:col10/1731962453190/Put/seqid=0 2024-11-18T20:40:55,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741884_1060 (size=12151) 2024-11-18T20:40:55,493 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/6f4a271719b5425ea874eb71e99e4a68 2024-11-18T20:40:55,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/497b14d533cb4a56bf8d100482542d0c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/497b14d533cb4a56bf8d100482542d0c 2024-11-18T20:40:55,506 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/497b14d533cb4a56bf8d100482542d0c, entries=150, sequenceid=214, filesize=11.9 K 2024-11-18T20:40:55,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/571d2bd8172c4aa19f9371f050a4f9b7 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/571d2bd8172c4aa19f9371f050a4f9b7 2024-11-18T20:40:55,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962515507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:55,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:55,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962515508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962515507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:55,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:55,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962515508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:55,514 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/571d2bd8172c4aa19f9371f050a4f9b7, entries=150, sequenceid=214, filesize=11.9 K 2024-11-18T20:40:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:55,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/6f4a271719b5425ea874eb71e99e4a68 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/6f4a271719b5425ea874eb71e99e4a68 2024-11-18T20:40:55,522 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/6f4a271719b5425ea874eb71e99e4a68, entries=150, sequenceid=214, filesize=11.9 K 2024-11-18T20:40:55,523 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4420199d334b1e279fd666dd4cb607ce in 1282ms, sequenceid=214, compaction requested=false 2024-11-18T20:40:55,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:55,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:55,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-18T20:40:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-18T20:40:55,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-18T20:40:55,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0560 sec 2024-11-18T20:40:55,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.0630 sec 2024-11-18T20:40:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T20:40:55,618 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:40:55,618 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:40:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-18T20:40:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T20:40:55,621 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:40:55,622 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:40:55,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:40:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T20:40:55,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-18T20:40:55,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:55,777 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T20:40:55,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:40:55,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:55,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:40:55,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:55,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:40:55,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:55,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/525be088e8b64a24b9ab4dc587f95e84 is 50, key is test_row_0/A:col10/1731962454364/Put/seqid=0 2024-11-18T20:40:55,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741885_1061 (size=12151) 2024-11-18T20:40:55,791 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/525be088e8b64a24b9ab4dc587f95e84 2024-11-18T20:40:55,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/067bb369719a4db9afd1ecf348071658 is 50, key is test_row_0/B:col10/1731962454364/Put/seqid=0 2024-11-18T20:40:55,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741886_1062 (size=12151) 2024-11-18T20:40:55,817 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/067bb369719a4db9afd1ecf348071658 2024-11-18T20:40:55,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/ed210d188de9478190e798f4101e97cb is 50, key is test_row_0/C:col10/1731962454364/Put/seqid=0 2024-11-18T20:40:55,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741887_1063 (size=12151) 2024-11-18T20:40:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T20:40:56,240 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/ed210d188de9478190e798f4101e97cb 2024-11-18T20:40:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T20:40:56,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/525be088e8b64a24b9ab4dc587f95e84 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/525be088e8b64a24b9ab4dc587f95e84 2024-11-18T20:40:56,256 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/525be088e8b64a24b9ab4dc587f95e84, entries=150, sequenceid=236, filesize=11.9 K 2024-11-18T20:40:56,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/067bb369719a4db9afd1ecf348071658 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/067bb369719a4db9afd1ecf348071658 2024-11-18T20:40:56,263 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/067bb369719a4db9afd1ecf348071658, entries=150, sequenceid=236, filesize=11.9 K 2024-11-18T20:40:56,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/ed210d188de9478190e798f4101e97cb as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/ed210d188de9478190e798f4101e97cb 2024-11-18T20:40:56,273 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/ed210d188de9478190e798f4101e97cb, entries=150, sequenceid=236, filesize=11.9 K 2024-11-18T20:40:56,274 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 4420199d334b1e279fd666dd4cb607ce in 498ms, sequenceid=236, compaction requested=true 2024-11-18T20:40:56,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:56,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:56,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-18T20:40:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-18T20:40:56,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-18T20:40:56,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 654 msec 2024-11-18T20:40:56,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 664 msec 2024-11-18T20:40:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:56,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T20:40:56,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:40:56,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:56,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:40:56,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:56,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:40:56,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:56,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/0255b1f1f51d4024ba116ced8e13d53f is 50, key is test_row_0/A:col10/1731962456532/Put/seqid=0 2024-11-18T20:40:56,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741888_1064 (size=12151) 2024-11-18T20:40:56,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/0255b1f1f51d4024ba116ced8e13d53f 2024-11-18T20:40:56,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/a34bf9023864427198d41360ae89cf11 is 50, key is test_row_0/B:col10/1731962456532/Put/seqid=0 2024-11-18T20:40:56,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741889_1065 (size=12151) 2024-11-18T20:40:56,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962516598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:56,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962516599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:56,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962516601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:56,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:56,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962516602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:56,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:56,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962516707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:56,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962516707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:56,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962516718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:56,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962516718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T20:40:56,758 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:40:56,758 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:40:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-18T20:40:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T20:40:56,760 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:40:56,761 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:40:56,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:40:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T20:40:56,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T20:40:56,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:56,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:56,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962516917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:56,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:56,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962516917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:56,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:56,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962516927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:56,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:56,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962516928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:56,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:56,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:56,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/a34bf9023864427198d41360ae89cf11 2024-11-18T20:40:57,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/2209763acde74a12b75a97d5184eec96 is 50, key is test_row_0/C:col10/1731962456532/Put/seqid=0 2024-11-18T20:40:57,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741890_1066 (size=12151) 2024-11-18T20:40:57,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T20:40:57,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:57,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:57,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:57,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:57,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T20:40:57,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T20:40:57,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:57,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:57,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:57,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:57,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:57,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:57,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962517227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731962517227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962517227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:57,233 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in 4420199d334b1e279fd666dd4cb607ce of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:40:57,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:57,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962517238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:57,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962517238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:57,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:57,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T20:40:57,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:57,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:57,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:57,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:57,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:40:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T20:40:57,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/2209763acde74a12b75a97d5184eec96 2024-11-18T20:40:57,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/0255b1f1f51d4024ba116ced8e13d53f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0255b1f1f51d4024ba116ced8e13d53f 2024-11-18T20:40:57,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0255b1f1f51d4024ba116ced8e13d53f, entries=150, sequenceid=248, filesize=11.9 K 2024-11-18T20:40:57,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/a34bf9023864427198d41360ae89cf11 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a34bf9023864427198d41360ae89cf11 2024-11-18T20:40:57,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a34bf9023864427198d41360ae89cf11, entries=150, sequenceid=248, filesize=11.9 K 2024-11-18T20:40:57,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/2209763acde74a12b75a97d5184eec96 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/2209763acde74a12b75a97d5184eec96 2024-11-18T20:40:57,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/2209763acde74a12b75a97d5184eec96, entries=150, sequenceid=248, filesize=11.9 K 2024-11-18T20:40:57,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4420199d334b1e279fd666dd4cb607ce in 877ms, sequenceid=248, compaction requested=true 2024-11-18T20:40:57,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:57,445 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:40:57,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:40:57,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:57,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:40:57,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:57,445 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:40:57,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:40:57,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:40:57,447 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:40:57,447 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:40:57,447 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/A is initiating minor compaction (all files) 2024-11-18T20:40:57,447 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/B is initiating minor compaction (all files) 2024-11-18T20:40:57,447 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/A in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:57,447 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/B in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:57,447 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/263f3d31bc494878a3be61551a8c5f5a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/497b14d533cb4a56bf8d100482542d0c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/525be088e8b64a24b9ab4dc587f95e84, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0255b1f1f51d4024ba116ced8e13d53f] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=47.9 K 2024-11-18T20:40:57,447 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f49ff82855434bf89c88b18d8e94c0a7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/571d2bd8172c4aa19f9371f050a4f9b7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/067bb369719a4db9afd1ecf348071658, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a34bf9023864427198d41360ae89cf11] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=47.9 K 2024-11-18T20:40:57,448 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting f49ff82855434bf89c88b18d8e94c0a7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731962453054 2024-11-18T20:40:57,448 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 263f3d31bc494878a3be61551a8c5f5a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731962453054 2024-11-18T20:40:57,448 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 497b14d533cb4a56bf8d100482542d0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731962453185 2024-11-18T20:40:57,449 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 525be088e8b64a24b9ab4dc587f95e84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731962454360 2024-11-18T20:40:57,449 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 571d2bd8172c4aa19f9371f050a4f9b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731962453185 2024-11-18T20:40:57,449 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0255b1f1f51d4024ba116ced8e13d53f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731962456532 2024-11-18T20:40:57,450 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 067bb369719a4db9afd1ecf348071658, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731962454360 2024-11-18T20:40:57,450 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting a34bf9023864427198d41360ae89cf11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731962456532 2024-11-18T20:40:57,465 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#A#compaction#54 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:40:57,466 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/fa477d133cee4d97886bef9655cab19f is 50, key is test_row_0/A:col10/1731962456532/Put/seqid=0 2024-11-18T20:40:57,472 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:40:57,473 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/07aff51ab31a4cfca93d46bfc98ac7d5 is 50, key is test_row_0/B:col10/1731962456532/Put/seqid=0 2024-11-18T20:40:57,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741891_1067 (size=12765) 2024-11-18T20:40:57,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741892_1068 (size=12765) 2024-11-18T20:40:57,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T20:40:57,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:57,529 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T20:40:57,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:40:57,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:57,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:40:57,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:57,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:40:57,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:57,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/5cf987695fda4a18a29b7ff35de8c136 is 50, key is test_row_0/A:col10/1731962456596/Put/seqid=0 2024-11-18T20:40:57,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741893_1069 (size=12301) 2024-11-18T20:40:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:57,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:57,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:57,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962517754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:57,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:57,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962517758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:57,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:57,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:57,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962517759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:57,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:57,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:57,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962517760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:57,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962517868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:57,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:57,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:57,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962517868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:57,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:57,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:57,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962517877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:57,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:57,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:57,897 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/fa477d133cee4d97886bef9655cab19f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/fa477d133cee4d97886bef9655cab19f 2024-11-18T20:40:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T20:40:57,908 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/07aff51ab31a4cfca93d46bfc98ac7d5 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/07aff51ab31a4cfca93d46bfc98ac7d5 2024-11-18T20:40:57,910 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/A of 4420199d334b1e279fd666dd4cb607ce into fa477d133cee4d97886bef9655cab19f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:40:57,910 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:57,910 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/A, priority=12, startTime=1731962457444; duration=0sec 2024-11-18T20:40:57,910 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:40:57,910 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:A 2024-11-18T20:40:57,910 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:40:57,917 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:40:57,917 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/C is initiating minor compaction (all files) 2024-11-18T20:40:57,917 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/C in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:57,917 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/c5f698e6f0a9443da482b7c9c2c2ae52, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/6f4a271719b5425ea874eb71e99e4a68, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/ed210d188de9478190e798f4101e97cb, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/2209763acde74a12b75a97d5184eec96] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=47.9 K 2024-11-18T20:40:57,918 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/B of 4420199d334b1e279fd666dd4cb607ce into 07aff51ab31a4cfca93d46bfc98ac7d5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:40:57,918 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:57,918 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/B, priority=12, startTime=1731962457445; duration=0sec 2024-11-18T20:40:57,918 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5f698e6f0a9443da482b7c9c2c2ae52, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731962453054 2024-11-18T20:40:57,918 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:57,918 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:B 2024-11-18T20:40:57,919 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f4a271719b5425ea874eb71e99e4a68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731962453185 2024-11-18T20:40:57,919 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed210d188de9478190e798f4101e97cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731962454360 2024-11-18T20:40:57,920 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2209763acde74a12b75a97d5184eec96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731962456532 2024-11-18T20:40:57,937 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#C#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:40:57,938 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/692c991977954a0593a604c56b142f77 is 50, key is test_row_0/C:col10/1731962456532/Put/seqid=0 2024-11-18T20:40:57,939 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/5cf987695fda4a18a29b7ff35de8c136 2024-11-18T20:40:57,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741894_1070 (size=12765) 2024-11-18T20:40:57,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/3bafab84d772478d8cd7799a1c5d3e22 is 50, key is test_row_0/B:col10/1731962456596/Put/seqid=0 2024-11-18T20:40:57,959 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/692c991977954a0593a604c56b142f77 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/692c991977954a0593a604c56b142f77 2024-11-18T20:40:57,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741895_1071 (size=12301) 2024-11-18T20:40:57,976 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/C of 4420199d334b1e279fd666dd4cb607ce into 692c991977954a0593a604c56b142f77(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:40:57,976 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:57,976 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/C, priority=12, startTime=1731962457445; duration=0sec 2024-11-18T20:40:57,976 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:57,976 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:C 2024-11-18T20:40:57,977 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/3bafab84d772478d8cd7799a1c5d3e22 2024-11-18T20:40:57,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/f2beb31af6254e7581168b8a1848d5d8 is 50, key is test_row_0/C:col10/1731962456596/Put/seqid=0 2024-11-18T20:40:57,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741896_1072 (size=12301) 2024-11-18T20:40:58,000 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/f2beb31af6254e7581168b8a1848d5d8 2024-11-18T20:40:58,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/5cf987695fda4a18a29b7ff35de8c136 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/5cf987695fda4a18a29b7ff35de8c136 2024-11-18T20:40:58,014 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/5cf987695fda4a18a29b7ff35de8c136, entries=150, sequenceid=268, filesize=12.0 K 2024-11-18T20:40:58,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/3bafab84d772478d8cd7799a1c5d3e22 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/3bafab84d772478d8cd7799a1c5d3e22 2024-11-18T20:40:58,022 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/3bafab84d772478d8cd7799a1c5d3e22, entries=150, sequenceid=268, filesize=12.0 K 2024-11-18T20:40:58,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/f2beb31af6254e7581168b8a1848d5d8 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/f2beb31af6254e7581168b8a1848d5d8 2024-11-18T20:40:58,031 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/f2beb31af6254e7581168b8a1848d5d8, entries=150, sequenceid=268, filesize=12.0 K 2024-11-18T20:40:58,032 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4420199d334b1e279fd666dd4cb607ce in 503ms, sequenceid=268, compaction requested=false 2024-11-18T20:40:58,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:58,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:58,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-18T20:40:58,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-18T20:40:58,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-18T20:40:58,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2750 sec 2024-11-18T20:40:58,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.2820 sec 2024-11-18T20:40:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:58,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T20:40:58,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:40:58,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:58,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:40:58,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:58,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:40:58,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:58,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/bb2683d7f86f483f93b29df87e94c699 is 50, key is test_row_0/A:col10/1731962457758/Put/seqid=0 2024-11-18T20:40:58,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741897_1073 (size=12301) 2024-11-18T20:40:58,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/bb2683d7f86f483f93b29df87e94c699 2024-11-18T20:40:58,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/288cb0c0e57a44d4ba6ee024e2114f86 is 50, key is test_row_0/B:col10/1731962457758/Put/seqid=0 2024-11-18T20:40:58,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741898_1074 (size=12301) 2024-11-18T20:40:58,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:58,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962518119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:58,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962518121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:58,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962518125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:58,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:58,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962518227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:58,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:58,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:58,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962518237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:58,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962518237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:58,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962518437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:58,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:58,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962518447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:58,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962518448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:58,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:58,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/288cb0c0e57a44d4ba6ee024e2114f86 2024-11-18T20:40:58,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/3eef767a82284b689ee260e45483937c is 50, key is test_row_0/C:col10/1731962457758/Put/seqid=0 2024-11-18T20:40:58,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741899_1075 (size=12301) 2024-11-18T20:40:58,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962518747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:58,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962518758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:58,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962518758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:58,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962518767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:58,770 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 4420199d334b1e279fd666dd4cb607ce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:40:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T20:40:58,918 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:40:58,918 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:40:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-18T20:40:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T20:40:58,921 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:40:58,922 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:40:58,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:40:58,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/3eef767a82284b689ee260e45483937c 2024-11-18T20:40:58,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/bb2683d7f86f483f93b29df87e94c699 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/bb2683d7f86f483f93b29df87e94c699 2024-11-18T20:40:58,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/bb2683d7f86f483f93b29df87e94c699, entries=150, sequenceid=284, filesize=12.0 K 2024-11-18T20:40:58,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/288cb0c0e57a44d4ba6ee024e2114f86 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/288cb0c0e57a44d4ba6ee024e2114f86 2024-11-18T20:40:58,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/288cb0c0e57a44d4ba6ee024e2114f86, entries=150, sequenceid=284, filesize=12.0 K 2024-11-18T20:40:58,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/3eef767a82284b689ee260e45483937c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/3eef767a82284b689ee260e45483937c 2024-11-18T20:40:58,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/3eef767a82284b689ee260e45483937c, entries=150, sequenceid=284, filesize=12.0 K 2024-11-18T20:40:58,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4420199d334b1e279fd666dd4cb607ce in 904ms, sequenceid=284, compaction requested=true 2024-11-18T20:40:58,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:58,986 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:40:58,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:40:58,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:58,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:40:58,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:58,986 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:40:58,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:40:58,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:40:58,987 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:40:58,987 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/A is initiating minor compaction (all files) 2024-11-18T20:40:58,987 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/A in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:58,987 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/fa477d133cee4d97886bef9655cab19f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/5cf987695fda4a18a29b7ff35de8c136, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/bb2683d7f86f483f93b29df87e94c699] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=36.5 K 2024-11-18T20:40:58,988 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:40:58,988 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa477d133cee4d97886bef9655cab19f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731962456532 2024-11-18T20:40:58,988 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/B is initiating minor compaction (all files) 2024-11-18T20:40:58,988 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/B in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:58,988 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/07aff51ab31a4cfca93d46bfc98ac7d5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/3bafab84d772478d8cd7799a1c5d3e22, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/288cb0c0e57a44d4ba6ee024e2114f86] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=36.5 K 2024-11-18T20:40:58,989 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 07aff51ab31a4cfca93d46bfc98ac7d5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731962456532 2024-11-18T20:40:58,989 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cf987695fda4a18a29b7ff35de8c136, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731962456594 2024-11-18T20:40:58,989 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb2683d7f86f483f93b29df87e94c699, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731962457740 2024-11-18T20:40:58,989 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bafab84d772478d8cd7799a1c5d3e22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731962456594 2024-11-18T20:40:58,989 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 288cb0c0e57a44d4ba6ee024e2114f86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731962457740 2024-11-18T20:40:59,003 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#B#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:40:59,003 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/a97f8491ea004aeba893afec1cdaf5bd is 50, key is test_row_0/B:col10/1731962457758/Put/seqid=0 2024-11-18T20:40:59,005 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#A#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:40:59,006 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/d1baf5f38ad1432ab4de5437facdd2da is 50, key is test_row_0/A:col10/1731962457758/Put/seqid=0 2024-11-18T20:40:59,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741901_1077 (size=13017) 2024-11-18T20:40:59,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T20:40:59,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741900_1076 (size=13017) 2024-11-18T20:40:59,037 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/d1baf5f38ad1432ab4de5437facdd2da as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/d1baf5f38ad1432ab4de5437facdd2da 2024-11-18T20:40:59,044 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/a97f8491ea004aeba893afec1cdaf5bd as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a97f8491ea004aeba893afec1cdaf5bd 2024-11-18T20:40:59,050 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/A of 4420199d334b1e279fd666dd4cb607ce into d1baf5f38ad1432ab4de5437facdd2da(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:40:59,050 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:59,051 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/A, priority=13, startTime=1731962458986; duration=0sec 2024-11-18T20:40:59,051 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:40:59,051 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:A 2024-11-18T20:40:59,051 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:40:59,053 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:40:59,053 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/C is initiating minor compaction (all files) 2024-11-18T20:40:59,053 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/C in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:59,054 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/692c991977954a0593a604c56b142f77, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/f2beb31af6254e7581168b8a1848d5d8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/3eef767a82284b689ee260e45483937c] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=36.5 K 2024-11-18T20:40:59,055 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 692c991977954a0593a604c56b142f77, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731962456532 2024-11-18T20:40:59,055 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2beb31af6254e7581168b8a1848d5d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731962456594 2024-11-18T20:40:59,056 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/B of 4420199d334b1e279fd666dd4cb607ce into a97f8491ea004aeba893afec1cdaf5bd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:40:59,056 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:59,056 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eef767a82284b689ee260e45483937c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731962457740 2024-11-18T20:40:59,056 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/B, priority=13, startTime=1731962458986; duration=0sec 2024-11-18T20:40:59,056 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:59,056 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:B 2024-11-18T20:40:59,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T20:40:59,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:59,075 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#C#compaction#65 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:40:59,075 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T20:40:59,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:40:59,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:59,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:40:59,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:59,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:40:59,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:59,076 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/a8a5ef3229f742509f4473e9e1f26627 is 50, key is test_row_0/C:col10/1731962457758/Put/seqid=0 2024-11-18T20:40:59,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/def1d608fa414d55b6b8134391620314 is 50, key is test_row_0/A:col10/1731962458119/Put/seqid=0 2024-11-18T20:40:59,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741902_1078 (size=13017) 2024-11-18T20:40:59,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741903_1079 (size=12301) 2024-11-18T20:40:59,105 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/def1d608fa414d55b6b8134391620314 2024-11-18T20:40:59,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/7940655c50004712a3ad12c6c2b0d979 is 50, key is test_row_0/B:col10/1731962458119/Put/seqid=0 2024-11-18T20:40:59,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741904_1080 (size=12301) 2024-11-18T20:40:59,150 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/7940655c50004712a3ad12c6c2b0d979 2024-11-18T20:40:59,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/163a62a66e2248e2975b9e9fb6895245 is 50, key is test_row_0/C:col10/1731962458119/Put/seqid=0 2024-11-18T20:40:59,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741905_1081 (size=12301) 2024-11-18T20:40:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T20:40:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:59,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:40:59,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962519288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:59,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:59,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962519290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:59,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962519293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:59,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:59,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:59,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962519397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:59,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:59,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:59,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962519407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962519408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:59,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:59,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:59,502 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/a8a5ef3229f742509f4473e9e1f26627 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/a8a5ef3229f742509f4473e9e1f26627 2024-11-18T20:40:59,511 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/C of 4420199d334b1e279fd666dd4cb607ce into a8a5ef3229f742509f4473e9e1f26627(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:40:59,511 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:59,511 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/C, priority=13, startTime=1731962458986; duration=0sec 2024-11-18T20:40:59,511 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:40:59,512 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:C 2024-11-18T20:40:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T20:40:59,582 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/163a62a66e2248e2975b9e9fb6895245 2024-11-18T20:40:59,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/def1d608fa414d55b6b8134391620314 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/def1d608fa414d55b6b8134391620314 2024-11-18T20:40:59,606 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/def1d608fa414d55b6b8134391620314, entries=150, sequenceid=305, filesize=12.0 K 2024-11-18T20:40:59,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/7940655c50004712a3ad12c6c2b0d979 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/7940655c50004712a3ad12c6c2b0d979 2024-11-18T20:40:59,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:59,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962519609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:59,617 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/7940655c50004712a3ad12c6c2b0d979, entries=150, sequenceid=305, filesize=12.0 K 2024-11-18T20:40:59,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/163a62a66e2248e2975b9e9fb6895245 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/163a62a66e2248e2975b9e9fb6895245 2024-11-18T20:40:59,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962519617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962519617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:59,643 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/163a62a66e2248e2975b9e9fb6895245, entries=150, sequenceid=305, filesize=12.0 K 2024-11-18T20:40:59,645 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4420199d334b1e279fd666dd4cb607ce in 570ms, sequenceid=305, compaction requested=false 2024-11-18T20:40:59,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:40:59,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:40:59,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-18T20:40:59,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-18T20:40:59,663 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-18T20:40:59,663 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 725 msec 2024-11-18T20:40:59,666 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 746 msec 2024-11-18T20:40:59,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T20:40:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:40:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:40:59,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:59,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:40:59,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:40:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:40:59,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/b7957e54f46c43f8af3dcad2ec5727ef is 50, key is test_row_0/A:col10/1731962459932/Put/seqid=0 2024-11-18T20:40:59,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962519983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:59,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:40:59,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:40:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962519993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:40:59,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:40:59,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:00,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741906_1082 (size=14741) 2024-11-18T20:41:00,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962519995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:00,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T20:41:00,057 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:00,058 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:41:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-18T20:41:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:41:00,061 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:41:00,062 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:41:00,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:41:00,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962520097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:00,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962520108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962520108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:41:00,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:41:00,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:00,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:00,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:00,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:00,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:00,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962520307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:00,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962520317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:00,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:00,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962520317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:00,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:41:00,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:00,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:00,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:00,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:00,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:41:00,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/b7957e54f46c43f8af3dcad2ec5727ef 2024-11-18T20:41:00,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/24387394210343eca5f5c26334ce32bb is 50, key is test_row_0/B:col10/1731962459932/Put/seqid=0 2024-11-18T20:41:00,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741907_1083 (size=12301) 2024-11-18T20:41:00,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:41:00,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:00,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:00,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:00,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:00,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:00,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962520617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:00,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962520627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:00,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962520627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:00,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:41:00,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:00,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:00,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:00,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:00,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:41:00,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962520778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:00,785 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 4420199d334b1e279fd666dd4cb607ce of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:00,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:41:00,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:00,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:00,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:00,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:00,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:00,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/24387394210343eca5f5c26334ce32bb 2024-11-18T20:41:00,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/0c21dd3675dc4c60a1c68617e1654a57 is 50, key is test_row_0/C:col10/1731962459932/Put/seqid=0 2024-11-18T20:41:00,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741908_1084 (size=12301) 2024-11-18T20:41:00,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:41:00,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:00,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:00,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:00,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:00,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:01,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962521138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:01,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962521139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:01,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:41:01,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:01,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:01,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:01,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:01,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:01,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962521149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:41:01,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/0c21dd3675dc4c60a1c68617e1654a57 2024-11-18T20:41:01,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:41:01,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:01,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:01,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:01,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:01,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:01,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/b7957e54f46c43f8af3dcad2ec5727ef as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/b7957e54f46c43f8af3dcad2ec5727ef 2024-11-18T20:41:01,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/b7957e54f46c43f8af3dcad2ec5727ef, entries=200, sequenceid=320, filesize=14.4 K 2024-11-18T20:41:01,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/24387394210343eca5f5c26334ce32bb as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/24387394210343eca5f5c26334ce32bb 2024-11-18T20:41:01,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/24387394210343eca5f5c26334ce32bb, entries=150, sequenceid=320, filesize=12.0 K 2024-11-18T20:41:01,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/0c21dd3675dc4c60a1c68617e1654a57 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/0c21dd3675dc4c60a1c68617e1654a57 2024-11-18T20:41:01,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/0c21dd3675dc4c60a1c68617e1654a57, entries=150, sequenceid=320, filesize=12.0 K 2024-11-18T20:41:01,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4420199d334b1e279fd666dd4cb607ce in 1422ms, sequenceid=320, compaction requested=true 2024-11-18T20:41:01,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:01,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:41:01,358 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:01,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:01,358 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:01,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:41:01,360 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:01,360 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/A is initiating minor compaction (all files) 2024-11-18T20:41:01,360 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/A in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:01,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:01,360 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/d1baf5f38ad1432ab4de5437facdd2da, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/def1d608fa414d55b6b8134391620314, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/b7957e54f46c43f8af3dcad2ec5727ef] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=39.1 K 2024-11-18T20:41:01,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:41:01,361 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:01,361 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/B is initiating minor compaction (all files) 2024-11-18T20:41:01,361 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/B in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:01,361 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a97f8491ea004aeba893afec1cdaf5bd, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/7940655c50004712a3ad12c6c2b0d979, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/24387394210343eca5f5c26334ce32bb] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=36.7 K 2024-11-18T20:41:01,362 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1baf5f38ad1432ab4de5437facdd2da, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731962457740 2024-11-18T20:41:01,363 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting a97f8491ea004aeba893afec1cdaf5bd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731962457740 2024-11-18T20:41:01,363 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting def1d608fa414d55b6b8134391620314, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731962458117 2024-11-18T20:41:01,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:01,364 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7957e54f46c43f8af3dcad2ec5727ef, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731962459274 2024-11-18T20:41:01,364 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 7940655c50004712a3ad12c6c2b0d979, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731962458117 2024-11-18T20:41:01,365 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 24387394210343eca5f5c26334ce32bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731962459277 2024-11-18T20:41:01,379 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:01,380 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/afcefd6477ee4e5fafd0a7643c983dae is 50, key is test_row_0/A:col10/1731962459932/Put/seqid=0 2024-11-18T20:41:01,383 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#B#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:01,384 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/9c1c6471af174333bf8523fc26e0905d is 50, key is test_row_0/B:col10/1731962459932/Put/seqid=0 2024-11-18T20:41:01,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741909_1085 (size=13119) 2024-11-18T20:41:01,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741910_1086 (size=13119) 2024-11-18T20:41:01,405 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/afcefd6477ee4e5fafd0a7643c983dae as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/afcefd6477ee4e5fafd0a7643c983dae 2024-11-18T20:41:01,411 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/9c1c6471af174333bf8523fc26e0905d as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/9c1c6471af174333bf8523fc26e0905d 2024-11-18T20:41:01,425 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/B of 4420199d334b1e279fd666dd4cb607ce into 9c1c6471af174333bf8523fc26e0905d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:01,425 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:01,425 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/B, priority=13, startTime=1731962461358; duration=0sec 2024-11-18T20:41:01,425 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:01,425 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:B 2024-11-18T20:41:01,425 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:01,426 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/A of 4420199d334b1e279fd666dd4cb607ce into afcefd6477ee4e5fafd0a7643c983dae(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:01,426 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:01,426 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/A, priority=13, startTime=1731962461357; duration=0sec 2024-11-18T20:41:01,426 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:01,426 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:A 2024-11-18T20:41:01,427 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:01,427 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/C is initiating minor compaction (all files) 2024-11-18T20:41:01,427 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/C in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:01,427 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/a8a5ef3229f742509f4473e9e1f26627, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/163a62a66e2248e2975b9e9fb6895245, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/0c21dd3675dc4c60a1c68617e1654a57] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=36.7 K 2024-11-18T20:41:01,428 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting a8a5ef3229f742509f4473e9e1f26627, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731962457740 2024-11-18T20:41:01,429 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 163a62a66e2248e2975b9e9fb6895245, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731962458117 2024-11-18T20:41:01,429 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c21dd3675dc4c60a1c68617e1654a57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731962459277 2024-11-18T20:41:01,451 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#C#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:01,452 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/a23fdc607665463b93b27f775d9af381 is 50, key is test_row_0/C:col10/1731962459932/Put/seqid=0 2024-11-18T20:41:01,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:41:01,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:01,460 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T20:41:01,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:41:01,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:01,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:41:01,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:01,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:41:01,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:01,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/f9c2799a52e4450186df2d902974fce3 is 50, key is test_row_0/A:col10/1731962459990/Put/seqid=0 2024-11-18T20:41:01,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741911_1087 (size=13119) 2024-11-18T20:41:01,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741912_1088 (size=12301) 2024-11-18T20:41:01,914 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/f9c2799a52e4450186df2d902974fce3 2024-11-18T20:41:01,918 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/a23fdc607665463b93b27f775d9af381 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/a23fdc607665463b93b27f775d9af381 2024-11-18T20:41:01,930 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/C of 4420199d334b1e279fd666dd4cb607ce into a23fdc607665463b93b27f775d9af381(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:01,931 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:01,931 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/C, priority=13, startTime=1731962461360; duration=0sec 2024-11-18T20:41:01,931 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:01,931 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:C 2024-11-18T20:41:01,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/f601bea6e780405a80afdd56eef3f6c6 is 50, key is test_row_0/B:col10/1731962459990/Put/seqid=0 2024-11-18T20:41:01,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741913_1089 (size=12301) 2024-11-18T20:41:01,955 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/f601bea6e780405a80afdd56eef3f6c6 2024-11-18T20:41:01,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/440b5b679aba4b6b8ae1fffac5e864b8 is 50, key is test_row_0/C:col10/1731962459990/Put/seqid=0 2024-11-18T20:41:01,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741914_1090 (size=12301) 2024-11-18T20:41:01,989 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/440b5b679aba4b6b8ae1fffac5e864b8 2024-11-18T20:41:01,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/f9c2799a52e4450186df2d902974fce3 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/f9c2799a52e4450186df2d902974fce3 2024-11-18T20:41:02,004 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/f9c2799a52e4450186df2d902974fce3, entries=150, sequenceid=340, filesize=12.0 K 2024-11-18T20:41:02,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/f601bea6e780405a80afdd56eef3f6c6 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f601bea6e780405a80afdd56eef3f6c6 2024-11-18T20:41:02,016 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f601bea6e780405a80afdd56eef3f6c6, entries=150, sequenceid=340, filesize=12.0 K 2024-11-18T20:41:02,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/440b5b679aba4b6b8ae1fffac5e864b8 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/440b5b679aba4b6b8ae1fffac5e864b8 2024-11-18T20:41:02,025 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/440b5b679aba4b6b8ae1fffac5e864b8, entries=150, sequenceid=340, filesize=12.0 K 2024-11-18T20:41:02,027 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 4420199d334b1e279fd666dd4cb607ce in 566ms, sequenceid=340, compaction requested=false 2024-11-18T20:41:02,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:02,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:02,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-18T20:41:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-18T20:41:02,032 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-18T20:41:02,032 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9660 sec 2024-11-18T20:41:02,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.9750 sec 2024-11-18T20:41:02,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T20:41:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:41:02,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:41:02,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:02,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:41:02,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:02,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:41:02,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:02,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/9d604b5a7faa4f1aaf2028dff1138457 is 50, key is test_row_0/A:col10/1731962462168/Put/seqid=0 2024-11-18T20:41:02,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741915_1091 (size=19621) 2024-11-18T20:41:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:41:02,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/9d604b5a7faa4f1aaf2028dff1138457 2024-11-18T20:41:02,219 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:02,220 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:41:02,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-18T20:41:02,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T20:41:02,223 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:41:02,225 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:41:02,225 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:41:02,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962462183, max timestamp=1731962462225Num uniques 859; 2024-11-18T20:41:02,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:41:02,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962462183, max timestamp=1731962462225Num uniques 859; 2024-11-18T20:41:02,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:41:02,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962462183, max timestamp=1731962462225Num uniques 859; 2024-11-18T20:41:02,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:41:02,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/6833ae27ebf54886ba6fb604dc93643c is 50, key is test_row_0/B:col10/1731962462168/Put/seqid=0 2024-11-18T20:41:02,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741916_1092 (size=12301) 2024-11-18T20:41:02,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/6833ae27ebf54886ba6fb604dc93643c 2024-11-18T20:41:02,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/66f713da0f6a416b9c0862af513e8b5e is 50, key is test_row_0/C:col10/1731962462168/Put/seqid=0 2024-11-18T20:41:02,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962522280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:02,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962522291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:02,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962522294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:02,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741917_1093 (size=12301) 2024-11-18T20:41:02,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/66f713da0f6a416b9c0862af513e8b5e 2024-11-18T20:41:02,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/9d604b5a7faa4f1aaf2028dff1138457 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/9d604b5a7faa4f1aaf2028dff1138457 2024-11-18T20:41:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T20:41:02,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/9d604b5a7faa4f1aaf2028dff1138457, entries=300, sequenceid=352, filesize=19.2 K 2024-11-18T20:41:02,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/6833ae27ebf54886ba6fb604dc93643c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/6833ae27ebf54886ba6fb604dc93643c 2024-11-18T20:41:02,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/6833ae27ebf54886ba6fb604dc93643c, entries=150, sequenceid=352, filesize=12.0 K 2024-11-18T20:41:02,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/66f713da0f6a416b9c0862af513e8b5e as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/66f713da0f6a416b9c0862af513e8b5e 2024-11-18T20:41:02,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/66f713da0f6a416b9c0862af513e8b5e, entries=150, sequenceid=352, filesize=12.0 K 2024-11-18T20:41:02,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4420199d334b1e279fd666dd4cb607ce in 196ms, sequenceid=352, compaction requested=true 2024-11-18T20:41:02,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:02,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:41:02,366 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:02,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:02,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:41:02,366 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:02,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:02,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:41:02,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:02,367 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45041 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:02,367 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/A is initiating minor compaction (all files) 2024-11-18T20:41:02,367 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/A in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:02,368 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/afcefd6477ee4e5fafd0a7643c983dae, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/f9c2799a52e4450186df2d902974fce3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/9d604b5a7faa4f1aaf2028dff1138457] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=44.0 K 2024-11-18T20:41:02,368 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:02,368 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/B is initiating minor compaction (all files) 2024-11-18T20:41:02,368 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/B in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:02,368 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/9c1c6471af174333bf8523fc26e0905d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f601bea6e780405a80afdd56eef3f6c6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/6833ae27ebf54886ba6fb604dc93643c] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=36.8 K 2024-11-18T20:41:02,368 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting afcefd6477ee4e5fafd0a7643c983dae, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731962459277 2024-11-18T20:41:02,369 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c1c6471af174333bf8523fc26e0905d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731962459277 2024-11-18T20:41:02,369 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting f601bea6e780405a80afdd56eef3f6c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731962459978 2024-11-18T20:41:02,369 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting f9c2799a52e4450186df2d902974fce3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731962459978 2024-11-18T20:41:02,370 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6833ae27ebf54886ba6fb604dc93643c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731962462162 2024-11-18T20:41:02,370 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d604b5a7faa4f1aaf2028dff1138457, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731962462160 2024-11-18T20:41:02,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-18T20:41:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:02,379 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-18T20:41:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:41:02,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:02,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:41:02,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:02,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:41:02,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:02,391 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#A#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:02,392 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/b20c5d4486dc457e8ae45d6a0aa1feca is 50, key is test_row_0/A:col10/1731962462168/Put/seqid=0 2024-11-18T20:41:02,400 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#B#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:02,401 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/2711a1b2dbaf4d2591aeb29759d53ab3 is 50, key is test_row_0/B:col10/1731962462168/Put/seqid=0 2024-11-18T20:41:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:41:02,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:02,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/9086d1a3c10d459787bd86837f434096 is 50, key is test_row_0/A:col10/1731962462288/Put/seqid=0 2024-11-18T20:41:02,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:02,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962522435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:02,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:02,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962522437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:02,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962522439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:02,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:02,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741918_1094 (size=13221) 2024-11-18T20:41:02,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741919_1095 (size=13221) 2024-11-18T20:41:02,474 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/b20c5d4486dc457e8ae45d6a0aa1feca as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/b20c5d4486dc457e8ae45d6a0aa1feca 2024-11-18T20:41:02,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741920_1096 (size=12301) 2024-11-18T20:41:02,487 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/9086d1a3c10d459787bd86837f434096 2024-11-18T20:41:02,490 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/A of 4420199d334b1e279fd666dd4cb607ce into b20c5d4486dc457e8ae45d6a0aa1feca(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:02,490 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:02,490 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/A, priority=13, startTime=1731962462365; duration=0sec 2024-11-18T20:41:02,491 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:02,491 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:A 2024-11-18T20:41:02,491 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:02,502 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:02,502 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/C is initiating minor compaction (all files) 2024-11-18T20:41:02,502 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/C in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:02,502 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/a23fdc607665463b93b27f775d9af381, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/440b5b679aba4b6b8ae1fffac5e864b8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/66f713da0f6a416b9c0862af513e8b5e] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=36.8 K 2024-11-18T20:41:02,503 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting a23fdc607665463b93b27f775d9af381, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731962459277 2024-11-18T20:41:02,503 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 440b5b679aba4b6b8ae1fffac5e864b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731962459978 2024-11-18T20:41:02,503 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 66f713da0f6a416b9c0862af513e8b5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731962462162 2024-11-18T20:41:02,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/6291a1455a5f4d1f8aae9d581d373f87 is 50, key is test_row_0/B:col10/1731962462288/Put/seqid=0 2024-11-18T20:41:02,520 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#C#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:02,521 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/74b228356b0e4008a44ab6fc573af1db is 50, key is test_row_0/C:col10/1731962462168/Put/seqid=0 2024-11-18T20:41:02,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T20:41:02,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741921_1097 (size=12301) 2024-11-18T20:41:02,553 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/6291a1455a5f4d1f8aae9d581d373f87 2024-11-18T20:41:02,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:02,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962522547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:02,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:02,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:02,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962522557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:02,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:02,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:02,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962522557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:02,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741922_1098 (size=13221) 2024-11-18T20:41:02,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/579de4adc1da4cb9b1065f6cd6046137 is 50, key is test_row_0/C:col10/1731962462288/Put/seqid=0 2024-11-18T20:41:02,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741923_1099 (size=12301) 2024-11-18T20:41:02,631 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/579de4adc1da4cb9b1065f6cd6046137 2024-11-18T20:41:02,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/9086d1a3c10d459787bd86837f434096 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/9086d1a3c10d459787bd86837f434096 2024-11-18T20:41:02,650 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/9086d1a3c10d459787bd86837f434096, entries=150, sequenceid=388, filesize=12.0 K 2024-11-18T20:41:02,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/6291a1455a5f4d1f8aae9d581d373f87 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/6291a1455a5f4d1f8aae9d581d373f87 2024-11-18T20:41:02,671 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/6291a1455a5f4d1f8aae9d581d373f87, entries=150, sequenceid=388, filesize=12.0 K 2024-11-18T20:41:02,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/579de4adc1da4cb9b1065f6cd6046137 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/579de4adc1da4cb9b1065f6cd6046137 2024-11-18T20:41:02,679 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/579de4adc1da4cb9b1065f6cd6046137, entries=150, sequenceid=388, filesize=12.0 K 2024-11-18T20:41:02,682 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 4420199d334b1e279fd666dd4cb607ce in 302ms, sequenceid=388, compaction requested=false 2024-11-18T20:41:02,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:02,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:02,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-18T20:41:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-18T20:41:02,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-18T20:41:02,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 459 msec 2024-11-18T20:41:02,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 467 msec 2024-11-18T20:41:02,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:41:02,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T20:41:02,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:41:02,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:02,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:41:02,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:02,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:41:02,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:02,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/cf829c02698e4027bbb2bab0b1cd1370 is 50, key is test_row_0/A:col10/1731962462781/Put/seqid=0 2024-11-18T20:41:02,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741924_1100 (size=14741) 2024-11-18T20:41:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T20:41:02,847 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:02,848 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:41:02,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-18T20:41:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T20:41:02,850 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:41:02,851 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:41:02,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:41:02,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962522848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:02,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:02,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962522850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:02,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962522854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:02,862 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/2711a1b2dbaf4d2591aeb29759d53ab3 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/2711a1b2dbaf4d2591aeb29759d53ab3 2024-11-18T20:41:02,872 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/B of 4420199d334b1e279fd666dd4cb607ce into 2711a1b2dbaf4d2591aeb29759d53ab3(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:02,875 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:02,875 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/B, priority=13, startTime=1731962462366; duration=0sec 2024-11-18T20:41:02,875 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:02,875 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:B 2024-11-18T20:41:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T20:41:02,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962522958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:02,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:02,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962522958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:02,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:02,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:02,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962522967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:02,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:02,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:02,997 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/74b228356b0e4008a44ab6fc573af1db as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/74b228356b0e4008a44ab6fc573af1db 2024-11-18T20:41:03,007 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/C of 4420199d334b1e279fd666dd4cb607ce into 74b228356b0e4008a44ab6fc573af1db(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:03,007 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:03,007 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/C, priority=13, startTime=1731962462366; duration=0sec 2024-11-18T20:41:03,007 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:03,007 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:C 2024-11-18T20:41:03,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-18T20:41:03,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:03,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:03,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:03,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:03,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:03,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-18T20:41:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:03,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T20:41:03,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962523168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:03,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:03,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962523169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:03,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:03,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962523177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:03,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/cf829c02698e4027bbb2bab0b1cd1370 2024-11-18T20:41:03,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/97dfd5d3ffd449759f0dcdad1f9c9fdb is 50, key is test_row_0/B:col10/1731962462781/Put/seqid=0 2024-11-18T20:41:03,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741925_1101 (size=12301) 2024-11-18T20:41:03,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/97dfd5d3ffd449759f0dcdad1f9c9fdb 2024-11-18T20:41:03,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/cf5a85e27bc345d68d0c9eda3d6dcce5 is 50, key is test_row_0/C:col10/1731962462781/Put/seqid=0 2024-11-18T20:41:03,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741926_1102 (size=12301) 2024-11-18T20:41:03,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/cf5a85e27bc345d68d0c9eda3d6dcce5 2024-11-18T20:41:03,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/cf829c02698e4027bbb2bab0b1cd1370 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/cf829c02698e4027bbb2bab0b1cd1370 2024-11-18T20:41:03,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-18T20:41:03,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:03,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:03,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:03,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:03,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:03,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/cf829c02698e4027bbb2bab0b1cd1370, entries=200, sequenceid=402, filesize=14.4 K 2024-11-18T20:41:03,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/97dfd5d3ffd449759f0dcdad1f9c9fdb as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/97dfd5d3ffd449759f0dcdad1f9c9fdb 2024-11-18T20:41:03,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/97dfd5d3ffd449759f0dcdad1f9c9fdb, entries=150, sequenceid=402, filesize=12.0 K 2024-11-18T20:41:03,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/cf5a85e27bc345d68d0c9eda3d6dcce5 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/cf5a85e27bc345d68d0c9eda3d6dcce5 2024-11-18T20:41:03,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/cf5a85e27bc345d68d0c9eda3d6dcce5, entries=150, sequenceid=402, filesize=12.0 K 2024-11-18T20:41:03,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4420199d334b1e279fd666dd4cb607ce in 561ms, sequenceid=402, compaction requested=true 2024-11-18T20:41:03,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:03,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:41:03,343 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:03,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:03,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:41:03,343 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:03,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:03,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:41:03,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:03,345 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:03,345 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/B is initiating minor compaction (all files) 2024-11-18T20:41:03,345 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/B in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:03,345 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/2711a1b2dbaf4d2591aeb29759d53ab3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/6291a1455a5f4d1f8aae9d581d373f87, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/97dfd5d3ffd449759f0dcdad1f9c9fdb] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=36.9 K 2024-11-18T20:41:03,346 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:03,346 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/A is initiating minor compaction (all files) 2024-11-18T20:41:03,346 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/A in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:03,346 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/b20c5d4486dc457e8ae45d6a0aa1feca, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/9086d1a3c10d459787bd86837f434096, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/cf829c02698e4027bbb2bab0b1cd1370] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=39.3 K 2024-11-18T20:41:03,346 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 2711a1b2dbaf4d2591aeb29759d53ab3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731962462162 2024-11-18T20:41:03,347 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 6291a1455a5f4d1f8aae9d581d373f87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731962462261 2024-11-18T20:41:03,347 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting b20c5d4486dc457e8ae45d6a0aa1feca, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731962462162 2024-11-18T20:41:03,347 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 97dfd5d3ffd449759f0dcdad1f9c9fdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731962462423 2024-11-18T20:41:03,351 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9086d1a3c10d459787bd86837f434096, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731962462261 2024-11-18T20:41:03,352 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf829c02698e4027bbb2bab0b1cd1370, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731962462423 2024-11-18T20:41:03,371 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#B#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:03,372 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/bd892cfd51064211a320fca97ef6a2ad is 50, key is test_row_0/B:col10/1731962462781/Put/seqid=0 2024-11-18T20:41:03,383 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#A#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:03,384 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/ef6a6ddae8804baf97e158c1f91e4347 is 50, key is test_row_0/A:col10/1731962462781/Put/seqid=0 2024-11-18T20:41:03,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741927_1103 (size=13323) 2024-11-18T20:41:03,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741928_1104 (size=13323) 2024-11-18T20:41:03,421 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/ef6a6ddae8804baf97e158c1f91e4347 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/ef6a6ddae8804baf97e158c1f91e4347 2024-11-18T20:41:03,433 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/A of 4420199d334b1e279fd666dd4cb607ce into ef6a6ddae8804baf97e158c1f91e4347(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:03,433 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:03,433 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/A, priority=13, startTime=1731962463343; duration=0sec 2024-11-18T20:41:03,436 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:03,436 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:A 2024-11-18T20:41:03,436 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:03,438 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:03,438 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/C is initiating minor compaction (all files) 2024-11-18T20:41:03,438 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/C in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:03,438 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/74b228356b0e4008a44ab6fc573af1db, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/579de4adc1da4cb9b1065f6cd6046137, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/cf5a85e27bc345d68d0c9eda3d6dcce5] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=36.9 K 2024-11-18T20:41:03,439 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74b228356b0e4008a44ab6fc573af1db, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731962462162 2024-11-18T20:41:03,439 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 579de4adc1da4cb9b1065f6cd6046137, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731962462261 2024-11-18T20:41:03,440 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf5a85e27bc345d68d0c9eda3d6dcce5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731962462423 2024-11-18T20:41:03,459 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#C#compaction#92 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:03,461 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/16ab0fb69c1c4ff8b841f4e63eb4257b is 50, key is test_row_0/C:col10/1731962462781/Put/seqid=0 2024-11-18T20:41:03,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-18T20:41:03,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:03,473 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T20:41:03,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:41:03,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:03,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:41:03,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:03,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:41:03,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T20:41:03,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:41:03,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/82c324f5661b4a54af32124d05ccf6ca is 50, key is test_row_0/A:col10/1731962462850/Put/seqid=0 2024-11-18T20:41:03,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741929_1105 (size=13323) 2024-11-18T20:41:03,515 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/16ab0fb69c1c4ff8b841f4e63eb4257b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/16ab0fb69c1c4ff8b841f4e63eb4257b 2024-11-18T20:41:03,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962523508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:03,524 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/C of 4420199d334b1e279fd666dd4cb607ce into 16ab0fb69c1c4ff8b841f4e63eb4257b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:03,524 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:03,524 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/C, priority=13, startTime=1731962463344; duration=0sec 2024-11-18T20:41:03,525 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:03,525 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:C 2024-11-18T20:41:03,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962523511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:03,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:03,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962523511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:03,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741930_1106 (size=12301) 2024-11-18T20:41:03,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962523627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:03,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:03,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962523637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:03,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962523637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:03,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:03,808 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/bd892cfd51064211a320fca97ef6a2ad as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/bd892cfd51064211a320fca97ef6a2ad 2024-11-18T20:41:03,818 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/B of 4420199d334b1e279fd666dd4cb607ce into bd892cfd51064211a320fca97ef6a2ad(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:03,818 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:03,818 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/B, priority=13, startTime=1731962463343; duration=0sec 2024-11-18T20:41:03,819 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:03,819 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:B 2024-11-18T20:41:03,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962523837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:03,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:03,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962523847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:03,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:03,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962523848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:03,949 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/82c324f5661b4a54af32124d05ccf6ca 2024-11-18T20:41:03,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/8052263f96ca49158cb6779f8cf797fc is 50, key is test_row_0/B:col10/1731962462850/Put/seqid=0 2024-11-18T20:41:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T20:41:03,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741931_1107 (size=12301) 2024-11-18T20:41:03,998 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/8052263f96ca49158cb6779f8cf797fc 2024-11-18T20:41:04,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/5f4cbf51e3f14f4b81f971800c2cf1dd is 50, key is test_row_0/C:col10/1731962462850/Put/seqid=0 2024-11-18T20:41:04,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741932_1108 (size=12301) 2024-11-18T20:41:04,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962524157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:04,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:04,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962524167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962524167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:04,421 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/5f4cbf51e3f14f4b81f971800c2cf1dd 2024-11-18T20:41:04,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/82c324f5661b4a54af32124d05ccf6ca as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/82c324f5661b4a54af32124d05ccf6ca 2024-11-18T20:41:04,444 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/82c324f5661b4a54af32124d05ccf6ca, entries=150, sequenceid=424, filesize=12.0 K 2024-11-18T20:41:04,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/8052263f96ca49158cb6779f8cf797fc as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/8052263f96ca49158cb6779f8cf797fc 2024-11-18T20:41:04,459 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/8052263f96ca49158cb6779f8cf797fc, entries=150, sequenceid=424, filesize=12.0 K 2024-11-18T20:41:04,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/5f4cbf51e3f14f4b81f971800c2cf1dd as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/5f4cbf51e3f14f4b81f971800c2cf1dd 2024-11-18T20:41:04,474 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/5f4cbf51e3f14f4b81f971800c2cf1dd, entries=150, sequenceid=424, filesize=12.0 K 2024-11-18T20:41:04,477 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4420199d334b1e279fd666dd4cb607ce in 1004ms, sequenceid=424, compaction requested=false 2024-11-18T20:41:04,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:04,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:04,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-18T20:41:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-18T20:41:04,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-18T20:41:04,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6290 sec 2024-11-18T20:41:04,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.6350 sec 2024-11-18T20:41:04,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T20:41:04,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:41:04,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:04,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:41:04,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:04,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:41:04,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:41:04,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/6290d143603b4b10aa56626cb2b162f1 is 50, key is test_row_0/A:col10/1731962463498/Put/seqid=0 2024-11-18T20:41:04,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741933_1109 (size=12301) 2024-11-18T20:41:04,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/6290d143603b4b10aa56626cb2b162f1 2024-11-18T20:41:04,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:04,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962524752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:04,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:04,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962524758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:04,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:04,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962524762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:04,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:04,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:04,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/ad09412dbcb4441998182c6634d7a013 is 50, key is test_row_0/B:col10/1731962463498/Put/seqid=0 2024-11-18T20:41:04,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741934_1110 (size=12301) 2024-11-18T20:41:04,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/ad09412dbcb4441998182c6634d7a013 2024-11-18T20:41:04,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:04,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731962524818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:04,828 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 4420199d334b1e279fd666dd4cb607ce of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:04,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/1eb1e9f302fd44d4aa696a459d0537f7 is 50, key is test_row_0/C:col10/1731962463498/Put/seqid=0 2024-11-18T20:41:04,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:04,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962524868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:04,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741935_1111 (size=12301) 2024-11-18T20:41:04,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/1eb1e9f302fd44d4aa696a459d0537f7 2024-11-18T20:41:04,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:04,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962524877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:04,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:04,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962524878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:04,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:04,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/6290d143603b4b10aa56626cb2b162f1 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/6290d143603b4b10aa56626cb2b162f1 2024-11-18T20:41:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:04,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/6290d143603b4b10aa56626cb2b162f1, entries=150, sequenceid=440, filesize=12.0 K 2024-11-18T20:41:04,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/ad09412dbcb4441998182c6634d7a013 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/ad09412dbcb4441998182c6634d7a013 2024-11-18T20:41:04,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/ad09412dbcb4441998182c6634d7a013, entries=150, sequenceid=440, filesize=12.0 K 2024-11-18T20:41:04,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/1eb1e9f302fd44d4aa696a459d0537f7 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/1eb1e9f302fd44d4aa696a459d0537f7 2024-11-18T20:41:04,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/1eb1e9f302fd44d4aa696a459d0537f7, entries=150, sequenceid=440, filesize=12.0 K 2024-11-18T20:41:04,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4420199d334b1e279fd666dd4cb607ce in 258ms, sequenceid=440, compaction requested=true 2024-11-18T20:41:04,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:04,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:41:04,931 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:04,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:04,932 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:04,933 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:04,933 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:04,933 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/B is initiating minor compaction (all files) 2024-11-18T20:41:04,933 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/A is initiating minor compaction (all files) 2024-11-18T20:41:04,934 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/B in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:04,934 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/A in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:04,934 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/bd892cfd51064211a320fca97ef6a2ad, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/8052263f96ca49158cb6779f8cf797fc, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/ad09412dbcb4441998182c6634d7a013] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=37.0 K 2024-11-18T20:41:04,934 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/ef6a6ddae8804baf97e158c1f91e4347, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/82c324f5661b4a54af32124d05ccf6ca, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/6290d143603b4b10aa56626cb2b162f1] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=37.0 K 2024-11-18T20:41:04,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:41:04,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:04,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4420199d334b1e279fd666dd4cb607ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:41:04,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:04,934 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef6a6ddae8804baf97e158c1f91e4347, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731962462423 2024-11-18T20:41:04,934 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting bd892cfd51064211a320fca97ef6a2ad, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731962462423 2024-11-18T20:41:04,935 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82c324f5661b4a54af32124d05ccf6ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731962462845 2024-11-18T20:41:04,935 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 8052263f96ca49158cb6779f8cf797fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731962462845 2024-11-18T20:41:04,935 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6290d143603b4b10aa56626cb2b162f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731962463498 2024-11-18T20:41:04,935 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting ad09412dbcb4441998182c6634d7a013, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731962463498 2024-11-18T20:41:04,959 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#A#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:04,960 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/e6f0f46eb28c43168c2b6320e9a041bd is 50, key is test_row_0/A:col10/1731962463498/Put/seqid=0 2024-11-18T20:41:04,978 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#B#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:04,979 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/6359c5bcbb6449eab9f1ed69bc6ced92 is 50, key is test_row_0/B:col10/1731962463498/Put/seqid=0 2024-11-18T20:41:04,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T20:41:04,998 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:04,999 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:41:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-18T20:41:05,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T20:41:05,002 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:41:05,003 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:41:05,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:41:05,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741936_1112 (size=13425) 2024-11-18T20:41:05,040 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/e6f0f46eb28c43168c2b6320e9a041bd as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/e6f0f46eb28c43168c2b6320e9a041bd 2024-11-18T20:41:05,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741937_1113 (size=13425) 2024-11-18T20:41:05,081 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/A of 4420199d334b1e279fd666dd4cb607ce into e6f0f46eb28c43168c2b6320e9a041bd(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:05,082 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:05,082 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/A, priority=13, startTime=1731962464931; duration=0sec 2024-11-18T20:41:05,082 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:05,082 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:A 2024-11-18T20:41:05,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T20:41:05,082 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:05,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:41:05,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:05,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:41:05,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:05,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:41:05,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:41:05,084 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:05,086 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 4420199d334b1e279fd666dd4cb607ce/C is initiating minor compaction (all files) 2024-11-18T20:41:05,086 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4420199d334b1e279fd666dd4cb607ce/C in TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:05,086 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/16ab0fb69c1c4ff8b841f4e63eb4257b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/5f4cbf51e3f14f4b81f971800c2cf1dd, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/1eb1e9f302fd44d4aa696a459d0537f7] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp, totalSize=37.0 K 2024-11-18T20:41:05,088 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16ab0fb69c1c4ff8b841f4e63eb4257b, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731962462423 2024-11-18T20:41:05,089 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f4cbf51e3f14f4b81f971800c2cf1dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731962462845 2024-11-18T20:41:05,090 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eb1e9f302fd44d4aa696a459d0537f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731962463498 2024-11-18T20:41:05,103 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4420199d334b1e279fd666dd4cb607ce#C#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:05,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/6b7d197e9a8e4f54a0f970906a3025c6 is 50, key is test_row_0/A:col10/1731962464755/Put/seqid=0 2024-11-18T20:41:05,104 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/b0232a2b2d4e40bfac0d9a420484a0b5 is 50, key is test_row_0/C:col10/1731962463498/Put/seqid=0 2024-11-18T20:41:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T20:41:05,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962525125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:05,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:05,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962525127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:05,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:05,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:05,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962525135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:05,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:05,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-18T20:41:05,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:05,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:05,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:05,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:05,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:05,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741938_1114 (size=13425) 2024-11-18T20:41:05,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741939_1115 (size=14741) 2024-11-18T20:41:05,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962525239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:05,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:05,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962525248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:05,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962525248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:05,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-18T20:41:05,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:05,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:05,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:05,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:05,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T20:41:05,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962525458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:05,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962525448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:05,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:05,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962525458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:05,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-18T20:41:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:05,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:05,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:05,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:05,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:05,480 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/6359c5bcbb6449eab9f1ed69bc6ced92 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/6359c5bcbb6449eab9f1ed69bc6ced92 2024-11-18T20:41:05,487 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/B of 4420199d334b1e279fd666dd4cb607ce into 6359c5bcbb6449eab9f1ed69bc6ced92(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:05,488 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:05,488 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/B, priority=13, startTime=1731962464932; duration=0sec 2024-11-18T20:41:05,488 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:05,488 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:B 2024-11-18T20:41:05,569 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/b0232a2b2d4e40bfac0d9a420484a0b5 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/b0232a2b2d4e40bfac0d9a420484a0b5 2024-11-18T20:41:05,579 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4420199d334b1e279fd666dd4cb607ce/C of 4420199d334b1e279fd666dd4cb607ce into b0232a2b2d4e40bfac0d9a420484a0b5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:05,579 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:05,579 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., storeName=4420199d334b1e279fd666dd4cb607ce/C, priority=13, startTime=1731962464934; duration=0sec 2024-11-18T20:41:05,579 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:05,579 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4420199d334b1e279fd666dd4cb607ce:C 2024-11-18T20:41:05,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/6b7d197e9a8e4f54a0f970906a3025c6 2024-11-18T20:41:05,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/a82039c08198428eabff3e521de21455 is 50, key is test_row_0/B:col10/1731962464755/Put/seqid=0 2024-11-18T20:41:05,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-18T20:41:05,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:05,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:05,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:05,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:05,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T20:41:05,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741940_1116 (size=12301) 2024-11-18T20:41:05,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39366 deadline: 1731962525767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:05,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1731962525767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:05,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-18T20:41:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:05,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:05,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39396 deadline: 1731962525777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:05,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:05,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:05,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-18T20:41:05,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:05,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. as already flushing 2024-11-18T20:41:05,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:05,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:05,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:05,999 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-18T20:41:05,999 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:41:05,999 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-18T20:41:05,999 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:41:06,004 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:06,005 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:06,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:06,005 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:06,005 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:06,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:06,006 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-18T20:41:06,007 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:41:06,007 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:06,007 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:06,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:06,011 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-18T20:41:06,011 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:41:06,011 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:06,011 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:06,011 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:06,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/a82039c08198428eabff3e521de21455 2024-11-18T20:41:06,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/ddf116aeae624c90ba3dfacb42cc25ac is 50, key is test_row_0/C:col10/1731962464755/Put/seqid=0 2024-11-18T20:41:06,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741941_1117 (size=12301) 2024-11-18T20:41:06,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/ddf116aeae624c90ba3dfacb42cc25ac 2024-11-18T20:41:06,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/6b7d197e9a8e4f54a0f970906a3025c6 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/6b7d197e9a8e4f54a0f970906a3025c6 2024-11-18T20:41:06,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/6b7d197e9a8e4f54a0f970906a3025c6, entries=200, sequenceid=460, filesize=14.4 K 2024-11-18T20:41:06,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/a82039c08198428eabff3e521de21455 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a82039c08198428eabff3e521de21455 2024-11-18T20:41:06,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a82039c08198428eabff3e521de21455, entries=150, sequenceid=460, filesize=12.0 K 2024-11-18T20:41:06,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/ddf116aeae624c90ba3dfacb42cc25ac as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/ddf116aeae624c90ba3dfacb42cc25ac 2024-11-18T20:41:06,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/ddf116aeae624c90ba3dfacb42cc25ac, entries=150, sequenceid=460, filesize=12.0 K 2024-11-18T20:41:06,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4420199d334b1e279fd666dd4cb607ce in 999ms, sequenceid=460, compaction requested=false 2024-11-18T20:41:06,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:06,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-18T20:41:06,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:06,084 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T20:41:06,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:41:06,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:06,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:41:06,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:06,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:41:06,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:06,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/9442cd505f4549eab591dc2a7d2023c9 is 50, key is test_row_1/A:col10/1731962465121/Put/seqid=0 2024-11-18T20:41:06,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741942_1118 (size=9857) 2024-11-18T20:41:06,102 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/9442cd505f4549eab591dc2a7d2023c9 2024-11-18T20:41:06,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/555a7e92e2504cc9be247d16e4f7096b is 50, key is test_row_1/B:col10/1731962465121/Put/seqid=0 2024-11-18T20:41:06,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741943_1119 (size=9857) 2024-11-18T20:41:06,120 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/555a7e92e2504cc9be247d16e4f7096b 2024-11-18T20:41:06,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/9be4aa7f62544a18a878dc50bbcda411 is 50, key is test_row_1/C:col10/1731962465121/Put/seqid=0 2024-11-18T20:41:06,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741944_1120 (size=9857) 2024-11-18T20:41:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T20:41:06,139 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/9be4aa7f62544a18a878dc50bbcda411 2024-11-18T20:41:06,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/9442cd505f4549eab591dc2a7d2023c9 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/9442cd505f4549eab591dc2a7d2023c9 2024-11-18T20:41:06,153 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/9442cd505f4549eab591dc2a7d2023c9, entries=100, sequenceid=475, filesize=9.6 K 2024-11-18T20:41:06,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/555a7e92e2504cc9be247d16e4f7096b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/555a7e92e2504cc9be247d16e4f7096b 2024-11-18T20:41:06,160 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/555a7e92e2504cc9be247d16e4f7096b, entries=100, sequenceid=475, filesize=9.6 K 2024-11-18T20:41:06,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/9be4aa7f62544a18a878dc50bbcda411 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/9be4aa7f62544a18a878dc50bbcda411 2024-11-18T20:41:06,172 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/9be4aa7f62544a18a878dc50bbcda411, entries=100, sequenceid=475, filesize=9.6 K 2024-11-18T20:41:06,173 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 4420199d334b1e279fd666dd4cb607ce in 90ms, sequenceid=475, compaction requested=true 2024-11-18T20:41:06,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 4420199d334b1e279fd666dd4cb607ce: 2024-11-18T20:41:06,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:06,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-18T20:41:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-18T20:41:06,178 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-18T20:41:06,178 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1710 sec 2024-11-18T20:41:06,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.1790 sec 2024-11-18T20:41:06,279 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-18T20:41:06,279 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:41:06,279 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:06,279 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:06,279 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:06,289 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-18T20:41:06,289 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-18T20:41:06,289 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:41:06,289 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:41:06,289 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:06,289 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:06,289 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:06,290 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:06,290 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:06,290 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T20:41:07,158 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:07,259 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-18T20:41:07,259 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:41:07,259 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:07,260 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:07,260 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:08,398 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:41:14,930 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-18T20:41:14,930 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:41:14,930 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:14,930 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:14,930 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:14,930 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T20:41:14,930 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-18T20:41:14,930 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-18T20:41:14,930 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-18T20:41:14,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-18T20:41:14,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-18T20:41:14,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T20:41:14,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2616 2024-11-18T20:41:14,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2582 2024-11-18T20:41:14,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T20:41:14,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1203 2024-11-18T20:41:14,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3605 rows 2024-11-18T20:41:14,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1211 2024-11-18T20:41:14,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3629 rows 2024-11-18T20:41:14,931 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:41:14,931 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:41:14,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:14,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:14,931 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:14,936 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T20:41:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T20:41:14,945 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962474945"}]},"ts":"1731962474945"} 2024-11-18T20:41:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-18T20:41:14,948 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T20:41:14,948 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T20:41:14,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:41:14,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4420199d334b1e279fd666dd4cb607ce, UNASSIGN}] 2024-11-18T20:41:14,959 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4420199d334b1e279fd666dd4cb607ce, UNASSIGN 2024-11-18T20:41:14,961 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=4420199d334b1e279fd666dd4cb607ce, regionState=CLOSING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:14,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4420199d334b1e279fd666dd4cb607ce, UNASSIGN because future has completed 2024-11-18T20:41:14,965 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:41:14,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:41:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-18T20:41:15,123 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:41:15,123 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:41:15,124 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 4420199d334b1e279fd666dd4cb607ce, disabling compactions & flushes 2024-11-18T20:41:15,124 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:15,124 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:15,124 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. after waiting 0 ms 2024-11-18T20:41:15,124 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:15,124 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 4420199d334b1e279fd666dd4cb607ce 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-18T20:41:15,125 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=A 2024-11-18T20:41:15,125 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:15,125 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=B 2024-11-18T20:41:15,125 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:15,125 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4420199d334b1e279fd666dd4cb607ce, store=C 2024-11-18T20:41:15,125 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:15,131 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/478898e4f49d4f78903e72e6dc4fe8dd is 50, key is test_row_0/A:col10/1731962467258/Put/seqid=0 2024-11-18T20:41:15,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741945_1121 (size=9857) 2024-11-18T20:41:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-18T20:41:15,538 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/478898e4f49d4f78903e72e6dc4fe8dd 2024-11-18T20:41:15,552 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/01ffe0da4c7245fa9427d99703e99694 is 50, key is test_row_0/B:col10/1731962467258/Put/seqid=0 2024-11-18T20:41:15,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741946_1122 (size=9857) 2024-11-18T20:41:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-18T20:41:15,960 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/01ffe0da4c7245fa9427d99703e99694 2024-11-18T20:41:15,970 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/3a30b312c0cb4cc994749034839ad351 is 50, key is test_row_0/C:col10/1731962467258/Put/seqid=0 2024-11-18T20:41:15,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741947_1123 (size=9857) 2024-11-18T20:41:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-18T20:41:16,376 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/3a30b312c0cb4cc994749034839ad351 2024-11-18T20:41:16,383 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/A/478898e4f49d4f78903e72e6dc4fe8dd as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/478898e4f49d4f78903e72e6dc4fe8dd 2024-11-18T20:41:16,389 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/478898e4f49d4f78903e72e6dc4fe8dd, entries=100, sequenceid=483, filesize=9.6 K 2024-11-18T20:41:16,390 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/B/01ffe0da4c7245fa9427d99703e99694 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/01ffe0da4c7245fa9427d99703e99694 2024-11-18T20:41:16,395 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/01ffe0da4c7245fa9427d99703e99694, entries=100, sequenceid=483, filesize=9.6 K 2024-11-18T20:41:16,395 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/.tmp/C/3a30b312c0cb4cc994749034839ad351 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/3a30b312c0cb4cc994749034839ad351 2024-11-18T20:41:16,400 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/3a30b312c0cb4cc994749034839ad351, entries=100, sequenceid=483, filesize=9.6 K 2024-11-18T20:41:16,401 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4420199d334b1e279fd666dd4cb607ce in 1277ms, sequenceid=483, compaction requested=true 2024-11-18T20:41:16,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/4af65aa116644db1b6b03be9846c609a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/102dd0b2bb4b4bad8c8dbc4a1e2fdb47, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/fe1591a6c1f0414ca3aa71d19edf30ca, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/e5cab452c6984fa585a3b094428b85f9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/04545e466a0f4b2f90478a5c05574eb6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/b5a0cabd58b24c20a5087d4b55032258, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0ce60dfbeb954e09a18f483b16ca945e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/57a4cee4b162495a83c5e5ca9d0e307f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0000e828360b46d3ac4d83b85c8cea0d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/daf3e7f7cff745e98f545f7f86d74cff, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/85f5ff16c9ad4272a63dd45cbc124067, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0230ecc5f5a044f4959a40d8d03c67ec, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/f7c9ccb5d82a4d33996f4df2d5b2b8c7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/30ac359f5f8146f49b9ab48daacec3e2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/263f3d31bc494878a3be61551a8c5f5a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/497b14d533cb4a56bf8d100482542d0c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/525be088e8b64a24b9ab4dc587f95e84, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/fa477d133cee4d97886bef9655cab19f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0255b1f1f51d4024ba116ced8e13d53f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/5cf987695fda4a18a29b7ff35de8c136, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/d1baf5f38ad1432ab4de5437facdd2da, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/bb2683d7f86f483f93b29df87e94c699, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/def1d608fa414d55b6b8134391620314, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/b7957e54f46c43f8af3dcad2ec5727ef, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/afcefd6477ee4e5fafd0a7643c983dae, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/f9c2799a52e4450186df2d902974fce3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/9d604b5a7faa4f1aaf2028dff1138457, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/b20c5d4486dc457e8ae45d6a0aa1feca, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/9086d1a3c10d459787bd86837f434096, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/cf829c02698e4027bbb2bab0b1cd1370, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/ef6a6ddae8804baf97e158c1f91e4347, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/82c324f5661b4a54af32124d05ccf6ca, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/6290d143603b4b10aa56626cb2b162f1] to archive 2024-11-18T20:41:16,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:41:16,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/4af65aa116644db1b6b03be9846c609a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/4af65aa116644db1b6b03be9846c609a 2024-11-18T20:41:16,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/102dd0b2bb4b4bad8c8dbc4a1e2fdb47 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/102dd0b2bb4b4bad8c8dbc4a1e2fdb47 2024-11-18T20:41:16,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/fe1591a6c1f0414ca3aa71d19edf30ca to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/fe1591a6c1f0414ca3aa71d19edf30ca 2024-11-18T20:41:16,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/e5cab452c6984fa585a3b094428b85f9 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/e5cab452c6984fa585a3b094428b85f9 2024-11-18T20:41:16,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/04545e466a0f4b2f90478a5c05574eb6 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/04545e466a0f4b2f90478a5c05574eb6 2024-11-18T20:41:16,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/b5a0cabd58b24c20a5087d4b55032258 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/b5a0cabd58b24c20a5087d4b55032258 2024-11-18T20:41:16,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0ce60dfbeb954e09a18f483b16ca945e to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0ce60dfbeb954e09a18f483b16ca945e 2024-11-18T20:41:16,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/57a4cee4b162495a83c5e5ca9d0e307f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/57a4cee4b162495a83c5e5ca9d0e307f 2024-11-18T20:41:16,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0000e828360b46d3ac4d83b85c8cea0d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0000e828360b46d3ac4d83b85c8cea0d 2024-11-18T20:41:16,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/daf3e7f7cff745e98f545f7f86d74cff to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/daf3e7f7cff745e98f545f7f86d74cff 2024-11-18T20:41:16,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/85f5ff16c9ad4272a63dd45cbc124067 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/85f5ff16c9ad4272a63dd45cbc124067 2024-11-18T20:41:16,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0230ecc5f5a044f4959a40d8d03c67ec to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0230ecc5f5a044f4959a40d8d03c67ec 2024-11-18T20:41:16,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/f7c9ccb5d82a4d33996f4df2d5b2b8c7 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/f7c9ccb5d82a4d33996f4df2d5b2b8c7 2024-11-18T20:41:16,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/30ac359f5f8146f49b9ab48daacec3e2 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/30ac359f5f8146f49b9ab48daacec3e2 2024-11-18T20:41:16,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/263f3d31bc494878a3be61551a8c5f5a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/263f3d31bc494878a3be61551a8c5f5a 2024-11-18T20:41:16,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/497b14d533cb4a56bf8d100482542d0c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/497b14d533cb4a56bf8d100482542d0c 2024-11-18T20:41:16,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/525be088e8b64a24b9ab4dc587f95e84 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/525be088e8b64a24b9ab4dc587f95e84 2024-11-18T20:41:16,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/fa477d133cee4d97886bef9655cab19f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/fa477d133cee4d97886bef9655cab19f 2024-11-18T20:41:16,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0255b1f1f51d4024ba116ced8e13d53f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/0255b1f1f51d4024ba116ced8e13d53f 2024-11-18T20:41:16,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/5cf987695fda4a18a29b7ff35de8c136 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/5cf987695fda4a18a29b7ff35de8c136 2024-11-18T20:41:16,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/d1baf5f38ad1432ab4de5437facdd2da to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/d1baf5f38ad1432ab4de5437facdd2da 2024-11-18T20:41:16,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/bb2683d7f86f483f93b29df87e94c699 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/bb2683d7f86f483f93b29df87e94c699 2024-11-18T20:41:16,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/def1d608fa414d55b6b8134391620314 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/def1d608fa414d55b6b8134391620314 2024-11-18T20:41:16,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/b7957e54f46c43f8af3dcad2ec5727ef to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/b7957e54f46c43f8af3dcad2ec5727ef 2024-11-18T20:41:16,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/afcefd6477ee4e5fafd0a7643c983dae to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/afcefd6477ee4e5fafd0a7643c983dae 2024-11-18T20:41:16,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/f9c2799a52e4450186df2d902974fce3 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/f9c2799a52e4450186df2d902974fce3 2024-11-18T20:41:16,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/9d604b5a7faa4f1aaf2028dff1138457 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/9d604b5a7faa4f1aaf2028dff1138457 2024-11-18T20:41:16,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/b20c5d4486dc457e8ae45d6a0aa1feca to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/b20c5d4486dc457e8ae45d6a0aa1feca 2024-11-18T20:41:16,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/9086d1a3c10d459787bd86837f434096 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/9086d1a3c10d459787bd86837f434096 2024-11-18T20:41:16,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/cf829c02698e4027bbb2bab0b1cd1370 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/cf829c02698e4027bbb2bab0b1cd1370 2024-11-18T20:41:16,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/ef6a6ddae8804baf97e158c1f91e4347 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/ef6a6ddae8804baf97e158c1f91e4347 2024-11-18T20:41:16,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/82c324f5661b4a54af32124d05ccf6ca to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/82c324f5661b4a54af32124d05ccf6ca 2024-11-18T20:41:16,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/6290d143603b4b10aa56626cb2b162f1 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/6290d143603b4b10aa56626cb2b162f1 2024-11-18T20:41:16,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/15b465e897ff406aad758581d2799675, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/139306673b1945f3ae8019906641aea4, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/24234c65e79c4efdb0ff4ea97f97116c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/dd4f8f0ed60f40828ed99be4b163bedf, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f2a1c83f0bed4334a3b978ab457575ac, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/42af97de7487424da6f38da4569bd638, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/64b1d8b97a5042068c49fe5b53e68a9b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f21c01d78cef4aebaa83fa1334e2b5dc, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a0909c66461e423b839dba3c6e2154ce, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/e028f39b46b34b6887feeea0291446c9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a2d6d235e84c4973a1085683e3e40ee0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/131b4c37965243cdb3d0db1929558c60, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/978f88640a6a463f82c77cc76a3d7f73, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f49ff82855434bf89c88b18d8e94c0a7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/8cdbd4fb1c80457eb826d95968c666e3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/571d2bd8172c4aa19f9371f050a4f9b7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/067bb369719a4db9afd1ecf348071658, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/07aff51ab31a4cfca93d46bfc98ac7d5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a34bf9023864427198d41360ae89cf11, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/3bafab84d772478d8cd7799a1c5d3e22, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a97f8491ea004aeba893afec1cdaf5bd, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/288cb0c0e57a44d4ba6ee024e2114f86, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/7940655c50004712a3ad12c6c2b0d979, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/9c1c6471af174333bf8523fc26e0905d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/24387394210343eca5f5c26334ce32bb, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f601bea6e780405a80afdd56eef3f6c6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/2711a1b2dbaf4d2591aeb29759d53ab3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/6833ae27ebf54886ba6fb604dc93643c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/6291a1455a5f4d1f8aae9d581d373f87, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/bd892cfd51064211a320fca97ef6a2ad, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/97dfd5d3ffd449759f0dcdad1f9c9fdb, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/8052263f96ca49158cb6779f8cf797fc, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/ad09412dbcb4441998182c6634d7a013] to archive 2024-11-18T20:41:16,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:41:16,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/15b465e897ff406aad758581d2799675 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/15b465e897ff406aad758581d2799675 2024-11-18T20:41:16,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/139306673b1945f3ae8019906641aea4 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/139306673b1945f3ae8019906641aea4 2024-11-18T20:41:16,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/24234c65e79c4efdb0ff4ea97f97116c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/24234c65e79c4efdb0ff4ea97f97116c 2024-11-18T20:41:16,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/dd4f8f0ed60f40828ed99be4b163bedf to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/dd4f8f0ed60f40828ed99be4b163bedf 2024-11-18T20:41:16,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f2a1c83f0bed4334a3b978ab457575ac to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f2a1c83f0bed4334a3b978ab457575ac 2024-11-18T20:41:16,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/42af97de7487424da6f38da4569bd638 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/42af97de7487424da6f38da4569bd638 2024-11-18T20:41:16,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/64b1d8b97a5042068c49fe5b53e68a9b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/64b1d8b97a5042068c49fe5b53e68a9b 2024-11-18T20:41:16,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f21c01d78cef4aebaa83fa1334e2b5dc to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f21c01d78cef4aebaa83fa1334e2b5dc 2024-11-18T20:41:16,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a0909c66461e423b839dba3c6e2154ce to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a0909c66461e423b839dba3c6e2154ce 2024-11-18T20:41:16,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/e028f39b46b34b6887feeea0291446c9 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/e028f39b46b34b6887feeea0291446c9 2024-11-18T20:41:16,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a2d6d235e84c4973a1085683e3e40ee0 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a2d6d235e84c4973a1085683e3e40ee0 2024-11-18T20:41:16,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/131b4c37965243cdb3d0db1929558c60 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/131b4c37965243cdb3d0db1929558c60 2024-11-18T20:41:16,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/978f88640a6a463f82c77cc76a3d7f73 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/978f88640a6a463f82c77cc76a3d7f73 2024-11-18T20:41:16,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f49ff82855434bf89c88b18d8e94c0a7 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f49ff82855434bf89c88b18d8e94c0a7 2024-11-18T20:41:16,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/8cdbd4fb1c80457eb826d95968c666e3 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/8cdbd4fb1c80457eb826d95968c666e3 2024-11-18T20:41:16,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/571d2bd8172c4aa19f9371f050a4f9b7 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/571d2bd8172c4aa19f9371f050a4f9b7 2024-11-18T20:41:16,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/067bb369719a4db9afd1ecf348071658 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/067bb369719a4db9afd1ecf348071658 2024-11-18T20:41:16,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/07aff51ab31a4cfca93d46bfc98ac7d5 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/07aff51ab31a4cfca93d46bfc98ac7d5 2024-11-18T20:41:16,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a34bf9023864427198d41360ae89cf11 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a34bf9023864427198d41360ae89cf11 2024-11-18T20:41:16,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/3bafab84d772478d8cd7799a1c5d3e22 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/3bafab84d772478d8cd7799a1c5d3e22 2024-11-18T20:41:16,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a97f8491ea004aeba893afec1cdaf5bd to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a97f8491ea004aeba893afec1cdaf5bd 2024-11-18T20:41:16,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/288cb0c0e57a44d4ba6ee024e2114f86 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/288cb0c0e57a44d4ba6ee024e2114f86 2024-11-18T20:41:16,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/7940655c50004712a3ad12c6c2b0d979 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/7940655c50004712a3ad12c6c2b0d979 2024-11-18T20:41:16,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/9c1c6471af174333bf8523fc26e0905d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/9c1c6471af174333bf8523fc26e0905d 2024-11-18T20:41:16,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/24387394210343eca5f5c26334ce32bb to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/24387394210343eca5f5c26334ce32bb 2024-11-18T20:41:16,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f601bea6e780405a80afdd56eef3f6c6 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/f601bea6e780405a80afdd56eef3f6c6 2024-11-18T20:41:16,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/2711a1b2dbaf4d2591aeb29759d53ab3 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/2711a1b2dbaf4d2591aeb29759d53ab3 2024-11-18T20:41:16,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/6833ae27ebf54886ba6fb604dc93643c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/6833ae27ebf54886ba6fb604dc93643c 2024-11-18T20:41:16,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/6291a1455a5f4d1f8aae9d581d373f87 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/6291a1455a5f4d1f8aae9d581d373f87 2024-11-18T20:41:16,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/bd892cfd51064211a320fca97ef6a2ad to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/bd892cfd51064211a320fca97ef6a2ad 2024-11-18T20:41:16,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/97dfd5d3ffd449759f0dcdad1f9c9fdb to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/97dfd5d3ffd449759f0dcdad1f9c9fdb 2024-11-18T20:41:16,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/8052263f96ca49158cb6779f8cf797fc to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/8052263f96ca49158cb6779f8cf797fc 2024-11-18T20:41:16,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/ad09412dbcb4441998182c6634d7a013 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/ad09412dbcb4441998182c6634d7a013 2024-11-18T20:41:16,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/30e522c075b44fd6a17380ec809dbcab, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/eade16475fbf4be39c2a09a734e1add5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/506b721b57814260889287c0c96432cd, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/cb212756686d4cbc9073be3f668009fc, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/f431af3539334f0392db951e2b1110e8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/7cfdbcc1f97d4a70adef0e12bb04e566, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/201d09dafac744cb84fc6e79a92c854b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/185f1ba088c041d18cc26efa0c2f1e71, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/5902745b6e7247f6a1c8e21da9bda59a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/cbf1d0d7e5f44c638959914076539d24, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/d7705e12e9c74bb4953c0b951b6ac035, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/db7d758a5d9540e8ab73b6b728783b16, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/7c24bf3d6df64f7d85daf5a09c36bf08, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/c5f698e6f0a9443da482b7c9c2c2ae52, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/e3159c52c8714038b7b10314ef262f4f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/6f4a271719b5425ea874eb71e99e4a68, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/ed210d188de9478190e798f4101e97cb, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/692c991977954a0593a604c56b142f77, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/2209763acde74a12b75a97d5184eec96, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/f2beb31af6254e7581168b8a1848d5d8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/a8a5ef3229f742509f4473e9e1f26627, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/3eef767a82284b689ee260e45483937c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/163a62a66e2248e2975b9e9fb6895245, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/a23fdc607665463b93b27f775d9af381, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/0c21dd3675dc4c60a1c68617e1654a57, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/440b5b679aba4b6b8ae1fffac5e864b8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/74b228356b0e4008a44ab6fc573af1db, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/66f713da0f6a416b9c0862af513e8b5e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/579de4adc1da4cb9b1065f6cd6046137, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/16ab0fb69c1c4ff8b841f4e63eb4257b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/cf5a85e27bc345d68d0c9eda3d6dcce5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/5f4cbf51e3f14f4b81f971800c2cf1dd, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/1eb1e9f302fd44d4aa696a459d0537f7] to archive 2024-11-18T20:41:16,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:41:16,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/30e522c075b44fd6a17380ec809dbcab to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/30e522c075b44fd6a17380ec809dbcab 2024-11-18T20:41:16,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/eade16475fbf4be39c2a09a734e1add5 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/eade16475fbf4be39c2a09a734e1add5 2024-11-18T20:41:16,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/506b721b57814260889287c0c96432cd to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/506b721b57814260889287c0c96432cd 2024-11-18T20:41:16,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/cb212756686d4cbc9073be3f668009fc to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/cb212756686d4cbc9073be3f668009fc 2024-11-18T20:41:16,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/f431af3539334f0392db951e2b1110e8 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/f431af3539334f0392db951e2b1110e8 2024-11-18T20:41:16,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/7cfdbcc1f97d4a70adef0e12bb04e566 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/7cfdbcc1f97d4a70adef0e12bb04e566 2024-11-18T20:41:16,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/201d09dafac744cb84fc6e79a92c854b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/201d09dafac744cb84fc6e79a92c854b 2024-11-18T20:41:16,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/185f1ba088c041d18cc26efa0c2f1e71 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/185f1ba088c041d18cc26efa0c2f1e71 2024-11-18T20:41:16,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/5902745b6e7247f6a1c8e21da9bda59a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/5902745b6e7247f6a1c8e21da9bda59a 2024-11-18T20:41:16,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/cbf1d0d7e5f44c638959914076539d24 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/cbf1d0d7e5f44c638959914076539d24 2024-11-18T20:41:16,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/d7705e12e9c74bb4953c0b951b6ac035 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/d7705e12e9c74bb4953c0b951b6ac035 2024-11-18T20:41:16,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/db7d758a5d9540e8ab73b6b728783b16 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/db7d758a5d9540e8ab73b6b728783b16 2024-11-18T20:41:16,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/7c24bf3d6df64f7d85daf5a09c36bf08 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/7c24bf3d6df64f7d85daf5a09c36bf08 2024-11-18T20:41:16,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/c5f698e6f0a9443da482b7c9c2c2ae52 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/c5f698e6f0a9443da482b7c9c2c2ae52 2024-11-18T20:41:16,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/e3159c52c8714038b7b10314ef262f4f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/e3159c52c8714038b7b10314ef262f4f 2024-11-18T20:41:16,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/6f4a271719b5425ea874eb71e99e4a68 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/6f4a271719b5425ea874eb71e99e4a68 2024-11-18T20:41:16,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/ed210d188de9478190e798f4101e97cb to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/ed210d188de9478190e798f4101e97cb 2024-11-18T20:41:16,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/692c991977954a0593a604c56b142f77 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/692c991977954a0593a604c56b142f77 2024-11-18T20:41:16,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/2209763acde74a12b75a97d5184eec96 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/2209763acde74a12b75a97d5184eec96 2024-11-18T20:41:16,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/f2beb31af6254e7581168b8a1848d5d8 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/f2beb31af6254e7581168b8a1848d5d8 2024-11-18T20:41:16,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/a8a5ef3229f742509f4473e9e1f26627 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/a8a5ef3229f742509f4473e9e1f26627 2024-11-18T20:41:16,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/3eef767a82284b689ee260e45483937c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/3eef767a82284b689ee260e45483937c 2024-11-18T20:41:16,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/163a62a66e2248e2975b9e9fb6895245 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/163a62a66e2248e2975b9e9fb6895245 2024-11-18T20:41:16,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/a23fdc607665463b93b27f775d9af381 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/a23fdc607665463b93b27f775d9af381 2024-11-18T20:41:16,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/0c21dd3675dc4c60a1c68617e1654a57 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/0c21dd3675dc4c60a1c68617e1654a57 2024-11-18T20:41:16,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/440b5b679aba4b6b8ae1fffac5e864b8 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/440b5b679aba4b6b8ae1fffac5e864b8 2024-11-18T20:41:16,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/74b228356b0e4008a44ab6fc573af1db to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/74b228356b0e4008a44ab6fc573af1db 2024-11-18T20:41:16,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/66f713da0f6a416b9c0862af513e8b5e to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/66f713da0f6a416b9c0862af513e8b5e 2024-11-18T20:41:16,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/579de4adc1da4cb9b1065f6cd6046137 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/579de4adc1da4cb9b1065f6cd6046137 2024-11-18T20:41:16,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/16ab0fb69c1c4ff8b841f4e63eb4257b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/16ab0fb69c1c4ff8b841f4e63eb4257b 2024-11-18T20:41:16,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/cf5a85e27bc345d68d0c9eda3d6dcce5 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/cf5a85e27bc345d68d0c9eda3d6dcce5 2024-11-18T20:41:16,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/5f4cbf51e3f14f4b81f971800c2cf1dd to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/5f4cbf51e3f14f4b81f971800c2cf1dd 2024-11-18T20:41:16,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/1eb1e9f302fd44d4aa696a459d0537f7 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/1eb1e9f302fd44d4aa696a459d0537f7 2024-11-18T20:41:16,570 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/recovered.edits/486.seqid, newMaxSeqId=486, maxSeqId=1 2024-11-18T20:41:16,571 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. 2024-11-18T20:41:16,571 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 4420199d334b1e279fd666dd4cb607ce: Waiting for close lock at 1731962475124Running coprocessor pre-close hooks at 1731962475124Disabling compacts and flushes for region at 1731962475124Disabling writes for close at 1731962475124Obtaining lock to block concurrent updates at 1731962475124Preparing flush snapshotting stores in 4420199d334b1e279fd666dd4cb607ce at 1731962475124Finished memstore snapshotting TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731962475125 (+1 ms)Flushing stores of TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce. at 1731962475126 (+1 ms)Flushing 4420199d334b1e279fd666dd4cb607ce/A: creating writer at 1731962475126Flushing 4420199d334b1e279fd666dd4cb607ce/A: appending metadata at 1731962475131 (+5 ms)Flushing 4420199d334b1e279fd666dd4cb607ce/A: closing flushed file at 1731962475131Flushing 4420199d334b1e279fd666dd4cb607ce/B: creating writer at 1731962475547 (+416 ms)Flushing 4420199d334b1e279fd666dd4cb607ce/B: appending metadata at 1731962475551 (+4 ms)Flushing 4420199d334b1e279fd666dd4cb607ce/B: closing flushed file at 1731962475551Flushing 4420199d334b1e279fd666dd4cb607ce/C: creating writer at 1731962475966 (+415 ms)Flushing 4420199d334b1e279fd666dd4cb607ce/C: appending metadata at 1731962475970 (+4 ms)Flushing 4420199d334b1e279fd666dd4cb607ce/C: closing flushed file at 1731962475970Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39bb4a13: reopening flushed file at 1731962476383 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f3ffa35: reopening flushed file at 1731962476389 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22507beb: reopening flushed file at 1731962476395 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4420199d334b1e279fd666dd4cb607ce in 1277ms, sequenceid=483, compaction requested=true at 1731962476401 (+6 ms)Writing region close event to WAL at 1731962476566 (+165 ms)Running coprocessor post-close hooks at 1731962476571 (+5 ms)Closed at 1731962476571 2024-11-18T20:41:16,574 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:41:16,574 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=4420199d334b1e279fd666dd4cb607ce, regionState=CLOSED 2024-11-18T20:41:16,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 because future has completed 2024-11-18T20:41:16,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-18T20:41:16,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 4420199d334b1e279fd666dd4cb607ce, server=c0a89b2656d4,45393,1731962440107 in 1.6130 sec 2024-11-18T20:41:16,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-18T20:41:16,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4420199d334b1e279fd666dd4cb607ce, UNASSIGN in 1.6230 sec 2024-11-18T20:41:16,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-18T20:41:16,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6340 sec 2024-11-18T20:41:16,588 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962476588"}]},"ts":"1731962476588"} 2024-11-18T20:41:16,590 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T20:41:16,590 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T20:41:16,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6520 sec 2024-11-18T20:41:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-18T20:41:17,098 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:17,105 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T20:41:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:41:17,114 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:41:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-18T20:41:17,116 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:41:17,122 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:41:17,127 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A, FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B, FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C, FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/recovered.edits] 2024-11-18T20:41:17,132 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/478898e4f49d4f78903e72e6dc4fe8dd to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/478898e4f49d4f78903e72e6dc4fe8dd 2024-11-18T20:41:17,134 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/6b7d197e9a8e4f54a0f970906a3025c6 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/6b7d197e9a8e4f54a0f970906a3025c6 2024-11-18T20:41:17,135 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/9442cd505f4549eab591dc2a7d2023c9 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/9442cd505f4549eab591dc2a7d2023c9 2024-11-18T20:41:17,137 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/e6f0f46eb28c43168c2b6320e9a041bd to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/A/e6f0f46eb28c43168c2b6320e9a041bd 2024-11-18T20:41:17,139 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/01ffe0da4c7245fa9427d99703e99694 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/01ffe0da4c7245fa9427d99703e99694 2024-11-18T20:41:17,140 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/555a7e92e2504cc9be247d16e4f7096b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/555a7e92e2504cc9be247d16e4f7096b 2024-11-18T20:41:17,142 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/6359c5bcbb6449eab9f1ed69bc6ced92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/6359c5bcbb6449eab9f1ed69bc6ced92 2024-11-18T20:41:17,143 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a82039c08198428eabff3e521de21455 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/B/a82039c08198428eabff3e521de21455 2024-11-18T20:41:17,145 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/3a30b312c0cb4cc994749034839ad351 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/3a30b312c0cb4cc994749034839ad351 2024-11-18T20:41:17,147 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/9be4aa7f62544a18a878dc50bbcda411 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/9be4aa7f62544a18a878dc50bbcda411 2024-11-18T20:41:17,148 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/b0232a2b2d4e40bfac0d9a420484a0b5 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/b0232a2b2d4e40bfac0d9a420484a0b5 2024-11-18T20:41:17,150 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/ddf116aeae624c90ba3dfacb42cc25ac to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/C/ddf116aeae624c90ba3dfacb42cc25ac 2024-11-18T20:41:17,154 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/recovered.edits/486.seqid to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce/recovered.edits/486.seqid 2024-11-18T20:41:17,155 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/4420199d334b1e279fd666dd4cb607ce 2024-11-18T20:41:17,155 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T20:41:17,161 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:41:17,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-18T20:41:17,171 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T20:41:17,199 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T20:41:17,201 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:41:17,201 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T20:41:17,201 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731962477201"}]},"ts":"9223372036854775807"} 2024-11-18T20:41:17,205 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T20:41:17,205 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4420199d334b1e279fd666dd4cb607ce, NAME => 'TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T20:41:17,206 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T20:41:17,206 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731962477206"}]},"ts":"9223372036854775807"} 2024-11-18T20:41:17,210 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T20:41:17,211 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:41:17,213 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 105 msec 2024-11-18T20:41:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-18T20:41:17,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T20:41:17,228 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:17,240 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=243 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_105021149_22 at /127.0.0.1:36872 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;c0a89b2656d4:45393-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=357 (was 277) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2918 (was 3635) 2024-11-18T20:41:17,251 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=243, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=357, ProcessCount=11, AvailableMemoryMB=2917 2024-11-18T20:41:17,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:41:17,253 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:41:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T20:41:17,256 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T20:41:17,256 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:17,256 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-18T20:41:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-18T20:41:17,257 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T20:41:17,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741948_1124 (size=1061) 2024-11-18T20:41:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-18T20:41:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-18T20:41:17,667 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093 2024-11-18T20:41:17,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741949_1125 (size=53) 2024-11-18T20:41:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-18T20:41:18,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:41:18,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 59378c222c2195896b2cd9530f63ae92, disabling compactions & flushes 2024-11-18T20:41:18,076 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:18,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:18,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. after waiting 0 ms 2024-11-18T20:41:18,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:18,076 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:18,077 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 59378c222c2195896b2cd9530f63ae92: Waiting for close lock at 1731962478076Disabling compacts and flushes for region at 1731962478076Disabling writes for close at 1731962478076Writing region close event to WAL at 1731962478076Closed at 1731962478076 2024-11-18T20:41:18,078 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T20:41:18,079 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731962478078"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731962478078"}]},"ts":"1731962478078"} 2024-11-18T20:41:18,081 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T20:41:18,082 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T20:41:18,083 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962478082"}]},"ts":"1731962478082"} 2024-11-18T20:41:18,085 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T20:41:18,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59378c222c2195896b2cd9530f63ae92, ASSIGN}] 2024-11-18T20:41:18,087 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59378c222c2195896b2cd9530f63ae92, ASSIGN 2024-11-18T20:41:18,088 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=59378c222c2195896b2cd9530f63ae92, ASSIGN; state=OFFLINE, location=c0a89b2656d4,45393,1731962440107; forceNewPlan=false, retain=false 2024-11-18T20:41:18,239 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=59378c222c2195896b2cd9530f63ae92, regionState=OPENING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:18,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=59378c222c2195896b2cd9530f63ae92, ASSIGN because future has completed 2024-11-18T20:41:18,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:41:18,397 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:18,397 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:41:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-18T20:41:18,398 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:18,398 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:41:18,398 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:18,398 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:18,400 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:18,401 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:41:18,402 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59378c222c2195896b2cd9530f63ae92 columnFamilyName A 2024-11-18T20:41:18,402 DEBUG [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:18,403 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] regionserver.HStore(327): Store=59378c222c2195896b2cd9530f63ae92/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:41:18,403 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:18,405 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:41:18,405 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59378c222c2195896b2cd9530f63ae92 columnFamilyName B 2024-11-18T20:41:18,405 DEBUG [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:18,405 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] regionserver.HStore(327): Store=59378c222c2195896b2cd9530f63ae92/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:41:18,405 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:18,406 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:41:18,407 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59378c222c2195896b2cd9530f63ae92 columnFamilyName C 2024-11-18T20:41:18,407 DEBUG [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:18,407 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] regionserver.HStore(327): Store=59378c222c2195896b2cd9530f63ae92/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:41:18,408 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:18,408 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:18,408 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:18,409 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:18,410 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:18,410 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:18,410 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:41:18,412 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:18,414 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:41:18,415 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 59378c222c2195896b2cd9530f63ae92; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70125953, jitterRate=0.04495812952518463}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:41:18,415 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:18,416 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 59378c222c2195896b2cd9530f63ae92: Running coprocessor pre-open hook at 1731962478398Writing region info on filesystem at 1731962478398Initializing all the Stores at 1731962478399 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962478399Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962478399Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962478399Cleaning up temporary data from old regions at 1731962478410 (+11 ms)Running coprocessor post-open hooks at 1731962478415 (+5 ms)Region opened successfully at 1731962478416 (+1 ms) 2024-11-18T20:41:18,416 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., pid=36, masterSystemTime=1731962478394 2024-11-18T20:41:18,418 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:18,418 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:18,419 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=59378c222c2195896b2cd9530f63ae92, regionState=OPEN, openSeqNum=2, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:18,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 because future has completed 2024-11-18T20:41:18,423 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-18T20:41:18,423 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 in 181 msec 2024-11-18T20:41:18,425 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-18T20:41:18,425 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59378c222c2195896b2cd9530f63ae92, ASSIGN in 338 msec 2024-11-18T20:41:18,426 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T20:41:18,426 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962478426"}]},"ts":"1731962478426"} 2024-11-18T20:41:18,428 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T20:41:18,428 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T20:41:18,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-11-18T20:41:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-18T20:41:19,408 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:19,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@168ecd40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:19,409 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:19,410 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:19,412 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:19,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:19,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:19,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6026b079, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:19,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:19,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:19,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:19,416 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63bdc2aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:19,418 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:19,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:19,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:19,424 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:41:19,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is c0a89b2656d4,46825,1731962439484 2024-11-18T20:41:19,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@47569338 2024-11-18T20:41:19,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:41:19,426 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:41:19,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:41:19,432 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:41:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-18T20:41:19,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741950_1126 (size=1097) 2024-11-18T20:41:19,854 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-18T20:41:19,855 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-18T20:41:19,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:41:19,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59378c222c2195896b2cd9530f63ae92, REOPEN/MOVE}] 2024-11-18T20:41:19,871 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59378c222c2195896b2cd9530f63ae92, REOPEN/MOVE 2024-11-18T20:41:19,872 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=59378c222c2195896b2cd9530f63ae92, regionState=CLOSING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:19,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=59378c222c2195896b2cd9530f63ae92, REOPEN/MOVE because future has completed 2024-11-18T20:41:19,875 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:41:19,876 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:41:20,030 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:20,030 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:41:20,030 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 59378c222c2195896b2cd9530f63ae92, disabling compactions & flushes 2024-11-18T20:41:20,030 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:20,030 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:20,030 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. after waiting 0 ms 2024-11-18T20:41:20,030 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:20,038 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T20:41:20,039 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:20,039 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 59378c222c2195896b2cd9530f63ae92: Waiting for close lock at 1731962480030Running coprocessor pre-close hooks at 1731962480030Disabling compacts and flushes for region at 1731962480030Disabling writes for close at 1731962480030Writing region close event to WAL at 1731962480031 (+1 ms)Running coprocessor post-close hooks at 1731962480039 (+8 ms)Closed at 1731962480039 2024-11-18T20:41:20,039 WARN [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 59378c222c2195896b2cd9530f63ae92 to self. 2024-11-18T20:41:20,042 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:20,043 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=59378c222c2195896b2cd9530f63ae92, regionState=CLOSED 2024-11-18T20:41:20,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 because future has completed 2024-11-18T20:41:20,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-18T20:41:20,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 in 171 msec 2024-11-18T20:41:20,051 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=59378c222c2195896b2cd9530f63ae92, REOPEN/MOVE; state=CLOSED, location=c0a89b2656d4,45393,1731962440107; forceNewPlan=false, retain=true 2024-11-18T20:41:20,202 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=59378c222c2195896b2cd9530f63ae92, regionState=OPENING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:20,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=59378c222c2195896b2cd9530f63ae92, REOPEN/MOVE because future has completed 2024-11-18T20:41:20,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:41:20,363 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:20,363 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:41:20,364 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:20,364 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:41:20,364 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:20,364 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:20,367 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:20,369 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:41:20,374 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59378c222c2195896b2cd9530f63ae92 columnFamilyName A 2024-11-18T20:41:20,376 DEBUG [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:20,377 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] regionserver.HStore(327): Store=59378c222c2195896b2cd9530f63ae92/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:41:20,378 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:20,379 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:41:20,379 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59378c222c2195896b2cd9530f63ae92 columnFamilyName B 2024-11-18T20:41:20,379 DEBUG [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:20,380 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] regionserver.HStore(327): Store=59378c222c2195896b2cd9530f63ae92/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:41:20,380 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:20,381 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:41:20,381 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59378c222c2195896b2cd9530f63ae92 columnFamilyName C 2024-11-18T20:41:20,381 DEBUG [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:20,382 INFO [StoreOpener-59378c222c2195896b2cd9530f63ae92-1 {}] regionserver.HStore(327): Store=59378c222c2195896b2cd9530f63ae92/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:41:20,382 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:20,382 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:20,383 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:20,384 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:20,385 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:20,385 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:20,386 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:41:20,387 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:20,388 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 59378c222c2195896b2cd9530f63ae92; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72200813, jitterRate=0.07587595283985138}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:41:20,388 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:20,390 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 59378c222c2195896b2cd9530f63ae92: Running coprocessor pre-open hook at 1731962480364Writing region info on filesystem at 1731962480364Initializing all the Stores at 1731962480365 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962480366 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962480366Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962480366Cleaning up temporary data from old regions at 1731962480385 (+19 ms)Running coprocessor post-open hooks at 1731962480388 (+3 ms)Region opened successfully at 1731962480390 (+2 ms) 2024-11-18T20:41:20,391 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., pid=41, masterSystemTime=1731962480358 2024-11-18T20:41:20,393 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:20,393 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:20,394 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=59378c222c2195896b2cd9530f63ae92, regionState=OPEN, openSeqNum=5, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:20,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 because future has completed 2024-11-18T20:41:20,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-18T20:41:20,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 in 194 msec 2024-11-18T20:41:20,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-18T20:41:20,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59378c222c2195896b2cd9530f63ae92, REOPEN/MOVE in 534 msec 2024-11-18T20:41:20,418 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-18T20:41:20,418 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 550 msec 2024-11-18T20:41:20,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 988 msec 2024-11-18T20:41:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-18T20:41:20,427 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:20,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@552fb1a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,428 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:20,428 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:20,429 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d0e0213, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:20,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:20,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:20,431 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f321aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18f01ca4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:20,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:20,433 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ba1929, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:20,436 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50306ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a283555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:20,438 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:20,439 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300f21f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:20,440 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b24fee3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ef6e97c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:20,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:20,444 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6361bc58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:20,446 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13f89be8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14984fac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,447 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:20,448 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:20,448 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e7550e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:20,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:20,450 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d458768, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a21ef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,452 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:20,453 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:20,453 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:20,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:20,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:20,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e6ad8e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:20,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:20,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:20,455 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17822dec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f62ef44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,456 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:20,457 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:20,457 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581aeddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:20,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:20,460 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40682de9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,462 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f60d5da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,462 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:20,462 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:20,463 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae94d25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:20,464 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:20,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c6f230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@139f617f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,466 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:20,466 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:20,467 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@217b1540, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:20,470 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:20,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75874e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:20,472 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:20,473 DEBUG [Thread-652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:20,473 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:20,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:20,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:20,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:20,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:20,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:20,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:20,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:20,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:20,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:20,476 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:20,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:20,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:20,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:41:20,480 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:20,480 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:20,481 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:20,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:20,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:41:20,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:20,483 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:41:20,484 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:41:20,483 DEBUG [Thread-650 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-18T20:41:20,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:41:20,486 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:41:20,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:41:20,487 DEBUG [Thread-648 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:20,488 DEBUG [Thread-648 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:20,488 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:20,488 DEBUG [Thread-643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:20,489 DEBUG [Thread-650 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:20,489 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:41:20,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:20,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:20,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:20,491 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:41:20,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:41:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:41:20,494 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:20,494 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:41:20,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:41:20,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:41:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:20,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T20:41:20,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:20,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:20,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:20,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:20,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:20,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:20,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182cabd711cd594239bc2f26d61bb31fa8_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962480487/Put/seqid=0 2024-11-18T20:41:20,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731962480527, max timestamp=1731962480567Num uniques 859; 2024-11-18T20:41:20,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T20:41:20,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962540582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:20,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731962480545, max timestamp=1731962480567Num uniques 859; 2024-11-18T20:41:20,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T20:41:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:20,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962480545, max timestamp=1731962480567Num uniques 859; 2024-11-18T20:41:20,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:41:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:41:20,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741951_1127 (size=19474) 2024-11-18T20:41:20,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962540619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:20,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962540620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:20,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731962540629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:20,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43748 deadline: 1731962540632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:20,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:20,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:41:20,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:20,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:20,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:20,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:20,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:20,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:20,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962540698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:20,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:20,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:20,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962540738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962540739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:20,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:20,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43748 deadline: 1731962540748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:20,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731962540748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:20,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:41:20,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:20,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:20,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:20,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:20,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:20,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:41:20,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962540917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:20,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:41:20,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:20,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:20,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:20,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:20,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:20,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962540957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:20,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962540958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43748 deadline: 1731962540958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:20,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731962540958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:21,016 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:21,026 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182cabd711cd594239bc2f26d61bb31fa8_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182cabd711cd594239bc2f26d61bb31fa8_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:21,028 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/35c9ae9b2533484098bfe6f5770aebaf, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:21,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/35c9ae9b2533484098bfe6f5770aebaf is 175, key is test_row_0/A:col10/1731962480487/Put/seqid=0 2024-11-18T20:41:21,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741952_1128 (size=56733) 2024-11-18T20:41:21,070 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/35c9ae9b2533484098bfe6f5770aebaf 2024-11-18T20:41:21,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:41:21,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:21,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:21,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:21,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:21,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:41:21,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/29d8933ecd6f46f98fbb442545cbbfd2 is 50, key is test_row_0/B:col10/1731962480487/Put/seqid=0 2024-11-18T20:41:21,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741953_1129 (size=12001) 2024-11-18T20:41:21,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/29d8933ecd6f46f98fbb442545cbbfd2 2024-11-18T20:41:21,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/a7b0195f582c4e0c9851cc6433716689 is 50, key is test_row_0/C:col10/1731962480487/Put/seqid=0 2024-11-18T20:41:21,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741954_1130 (size=12001) 2024-11-18T20:41:21,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/a7b0195f582c4e0c9851cc6433716689 2024-11-18T20:41:21,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/35c9ae9b2533484098bfe6f5770aebaf as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/35c9ae9b2533484098bfe6f5770aebaf 2024-11-18T20:41:21,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/35c9ae9b2533484098bfe6f5770aebaf, entries=300, sequenceid=15, filesize=55.4 K 2024-11-18T20:41:21,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/29d8933ecd6f46f98fbb442545cbbfd2 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/29d8933ecd6f46f98fbb442545cbbfd2 2024-11-18T20:41:21,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962541227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:21,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:21,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/29d8933ecd6f46f98fbb442545cbbfd2, entries=150, sequenceid=15, filesize=11.7 K 2024-11-18T20:41:21,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/a7b0195f582c4e0c9851cc6433716689 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/a7b0195f582c4e0c9851cc6433716689 2024-11-18T20:41:21,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/a7b0195f582c4e0c9851cc6433716689, entries=150, sequenceid=15, filesize=11.7 K 2024-11-18T20:41:21,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=124.84 KB/127835 for 59378c222c2195896b2cd9530f63ae92 in 747ms, sequenceid=15, compaction requested=false 2024-11-18T20:41:21,249 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T20:41:21,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:21,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:41:21,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:21,269 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=124.84 KB heapSize=354.38 KB 2024-11-18T20:41:21,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:21,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:21,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:21,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:21,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:21,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:21,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:21,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:21,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a84badb7f31841548e743ada8f3769a8_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962480628/Put/seqid=0 2024-11-18T20:41:21,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741955_1131 (size=12154) 2024-11-18T20:41:21,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:21,337 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a84badb7f31841548e743ada8f3769a8_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a84badb7f31841548e743ada8f3769a8_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:21,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/abc70358821543959318894ffe9156a9, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:21,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962541327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:21,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43748 deadline: 1731962541332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:21,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/abc70358821543959318894ffe9156a9 is 175, key is test_row_0/A:col10/1731962480628/Put/seqid=0 2024-11-18T20:41:21,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731962541340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:21,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:21,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:21,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962541345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:21,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741956_1132 (size=30955) 2024-11-18T20:41:21,368 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/abc70358821543959318894ffe9156a9 2024-11-18T20:41:21,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/5ff216eca128476b923e857616a0cd1a is 50, key is test_row_0/B:col10/1731962480628/Put/seqid=0 2024-11-18T20:41:21,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741957_1133 (size=12001) 2024-11-18T20:41:21,401 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/5ff216eca128476b923e857616a0cd1a 2024-11-18T20:41:21,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/1cb0368073a8456eac2ecfc4e7f9d648 is 50, key is test_row_0/C:col10/1731962480628/Put/seqid=0 2024-11-18T20:41:21,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741958_1134 (size=12001) 2024-11-18T20:41:21,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962541459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:21,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:21,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731962541460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:21,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43748 deadline: 1731962541465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:21,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962541467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:21,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:41:21,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962541677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:21,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:21,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731962541689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:21,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43748 deadline: 1731962541689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:21,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962541689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:21,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962541748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:21,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:21,841 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/1cb0368073a8456eac2ecfc4e7f9d648 2024-11-18T20:41:21,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/abc70358821543959318894ffe9156a9 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/abc70358821543959318894ffe9156a9 2024-11-18T20:41:21,866 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/abc70358821543959318894ffe9156a9, entries=150, sequenceid=50, filesize=30.2 K 2024-11-18T20:41:21,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/5ff216eca128476b923e857616a0cd1a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5ff216eca128476b923e857616a0cd1a 2024-11-18T20:41:21,884 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5ff216eca128476b923e857616a0cd1a, entries=150, sequenceid=50, filesize=11.7 K 2024-11-18T20:41:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/1cb0368073a8456eac2ecfc4e7f9d648 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/1cb0368073a8456eac2ecfc4e7f9d648 2024-11-18T20:41:21,897 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/1cb0368073a8456eac2ecfc4e7f9d648, entries=150, sequenceid=50, filesize=11.7 K 2024-11-18T20:41:21,898 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~124.84 KB/127835, heapSize ~354.34 KB/362840, currentSize=53.67 KB/54960 for 59378c222c2195896b2cd9530f63ae92 in 629ms, sequenceid=50, compaction requested=false 2024-11-18T20:41:21,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:21,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:21,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-18T20:41:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-18T20:41:21,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-18T20:41:21,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4100 sec 2024-11-18T20:41:21,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.4240 sec 2024-11-18T20:41:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:22,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T20:41:22,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:22,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:22,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:22,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:22,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:22,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:22,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183e2992c6d43144e78dbd9f11e2628ac0_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962481999/Put/seqid=0 2024-11-18T20:41:22,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962482022, max timestamp=1731962482042Num uniques 859; 2024-11-18T20:41:22,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:41:22,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962482022, max timestamp=1731962482042Num uniques 859; 2024-11-18T20:41:22,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:41:22,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962482022, max timestamp=1731962482042Num uniques 859; 2024-11-18T20:41:22,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:41:22,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741959_1135 (size=14594) 2024-11-18T20:41:22,072 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:22,080 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183e2992c6d43144e78dbd9f11e2628ac0_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183e2992c6d43144e78dbd9f11e2628ac0_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:22,081 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/7125a11bddb6470394e53ab2adb065f6, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:22,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/7125a11bddb6470394e53ab2adb065f6 is 175, key is test_row_0/A:col10/1731962481999/Put/seqid=0 2024-11-18T20:41:22,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962542091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43748 deadline: 1731962542091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731962542092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962542093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741960_1136 (size=39549) 2024-11-18T20:41:22,109 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/7125a11bddb6470394e53ab2adb065f6 2024-11-18T20:41:22,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/3873d2922e4d4e7b9f476af5ff04ef33 is 50, key is test_row_0/B:col10/1731962481999/Put/seqid=0 2024-11-18T20:41:22,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741961_1137 (size=12001) 2024-11-18T20:41:22,150 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T20:41:22,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962542207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962542208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731962542209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43748 deadline: 1731962542209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962542417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962542428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43748 deadline: 1731962542428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731962542428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/3873d2922e4d4e7b9f476af5ff04ef33 2024-11-18T20:41:22,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/1c768011f617403a9acf28845fda59e1 is 50, key is test_row_0/C:col10/1731962481999/Put/seqid=0 2024-11-18T20:41:22,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741962_1138 (size=12001) 2024-11-18T20:41:22,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/1c768011f617403a9acf28845fda59e1 2024-11-18T20:41:22,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/7125a11bddb6470394e53ab2adb065f6 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/7125a11bddb6470394e53ab2adb065f6 2024-11-18T20:41:22,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/7125a11bddb6470394e53ab2adb065f6, entries=200, sequenceid=62, filesize=38.6 K 2024-11-18T20:41:22,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/3873d2922e4d4e7b9f476af5ff04ef33 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3873d2922e4d4e7b9f476af5ff04ef33 2024-11-18T20:41:22,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3873d2922e4d4e7b9f476af5ff04ef33, entries=150, sequenceid=62, filesize=11.7 K 2024-11-18T20:41:22,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/1c768011f617403a9acf28845fda59e1 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/1c768011f617403a9acf28845fda59e1 2024-11-18T20:41:22,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/1c768011f617403a9acf28845fda59e1, entries=150, sequenceid=62, filesize=11.7 K 2024-11-18T20:41:22,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 59378c222c2195896b2cd9530f63ae92 in 619ms, sequenceid=62, compaction requested=true 2024-11-18T20:41:22,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:22,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:41:22,621 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:22,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:22,621 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:22,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:41:22,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:22,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:41:22,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:22,623 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:22,623 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127237 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:22,623 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/B is initiating minor compaction (all files) 2024-11-18T20:41:22,623 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/A is initiating minor compaction (all files) 2024-11-18T20:41:22,623 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/A in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:22,623 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/B in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:22,623 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/29d8933ecd6f46f98fbb442545cbbfd2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5ff216eca128476b923e857616a0cd1a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3873d2922e4d4e7b9f476af5ff04ef33] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=35.2 K 2024-11-18T20:41:22,623 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/35c9ae9b2533484098bfe6f5770aebaf, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/abc70358821543959318894ffe9156a9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/7125a11bddb6470394e53ab2adb065f6] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=124.3 K 2024-11-18T20:41:22,624 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:22,624 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/35c9ae9b2533484098bfe6f5770aebaf, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/abc70358821543959318894ffe9156a9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/7125a11bddb6470394e53ab2adb065f6] 2024-11-18T20:41:22,624 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 29d8933ecd6f46f98fbb442545cbbfd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731962480487 2024-11-18T20:41:22,624 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ff216eca128476b923e857616a0cd1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731962480600 2024-11-18T20:41:22,624 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35c9ae9b2533484098bfe6f5770aebaf, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731962480487 2024-11-18T20:41:22,625 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 3873d2922e4d4e7b9f476af5ff04ef33, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731962481326 2024-11-18T20:41:22,625 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting abc70358821543959318894ffe9156a9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731962480600 2024-11-18T20:41:22,625 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7125a11bddb6470394e53ab2adb065f6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731962481326 2024-11-18T20:41:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:41:22,648 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:22,648 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:41:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-18T20:41:22,652 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#B#compaction#120 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:22,653 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/4647cae4ae6b46a491138a6087d759ae is 50, key is test_row_0/B:col10/1731962481999/Put/seqid=0 2024-11-18T20:41:22,656 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:22,656 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:41:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:41:22,661 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:41:22,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:41:22,670 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118a21375f54f0641a48e1c0b677e3f712f_59378c222c2195896b2cd9530f63ae92 store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:22,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741963_1139 (size=12104) 2024-11-18T20:41:22,676 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118a21375f54f0641a48e1c0b677e3f712f_59378c222c2195896b2cd9530f63ae92, store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:22,676 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a21375f54f0641a48e1c0b677e3f712f_59378c222c2195896b2cd9530f63ae92 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:22,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741964_1140 (size=4469) 2024-11-18T20:41:22,687 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/4647cae4ae6b46a491138a6087d759ae as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/4647cae4ae6b46a491138a6087d759ae 2024-11-18T20:41:22,688 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#A#compaction#121 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:22,689 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/d32f35e5980f47ec88b01cb26462cbab is 175, key is test_row_0/A:col10/1731962481999/Put/seqid=0 2024-11-18T20:41:22,697 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/B of 59378c222c2195896b2cd9530f63ae92 into 4647cae4ae6b46a491138a6087d759ae(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:22,697 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:22,697 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/B, priority=13, startTime=1731962482621; duration=0sec 2024-11-18T20:41:22,697 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:22,697 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:B 2024-11-18T20:41:22,697 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:22,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741965_1141 (size=31058) 2024-11-18T20:41:22,707 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:22,708 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/C is initiating minor compaction (all files) 2024-11-18T20:41:22,708 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/C in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:22,708 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/a7b0195f582c4e0c9851cc6433716689, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/1cb0368073a8456eac2ecfc4e7f9d648, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/1c768011f617403a9acf28845fda59e1] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=35.2 K 2024-11-18T20:41:22,710 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting a7b0195f582c4e0c9851cc6433716689, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731962480487 2024-11-18T20:41:22,714 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cb0368073a8456eac2ecfc4e7f9d648, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731962480600 2024-11-18T20:41:22,717 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c768011f617403a9acf28845fda59e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731962481326 2024-11-18T20:41:22,728 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/d32f35e5980f47ec88b01cb26462cbab as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/d32f35e5980f47ec88b01cb26462cbab 2024-11-18T20:41:22,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-18T20:41:22,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:22,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:22,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:22,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:22,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:22,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:22,742 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/A of 59378c222c2195896b2cd9530f63ae92 into d32f35e5980f47ec88b01cb26462cbab(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:22,742 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:22,742 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/A, priority=13, startTime=1731962482621; duration=0sec 2024-11-18T20:41:22,742 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:22,742 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:A 2024-11-18T20:41:22,761 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#C#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:22,761 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/54193b2804d74d2b8964b52382458087 is 50, key is test_row_0/C:col10/1731962481999/Put/seqid=0 2024-11-18T20:41:22,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962542750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189a431d7515e74fd1b2b51c2cff0ce237_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962482729/Put/seqid=0 2024-11-18T20:41:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962542757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,764 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 59378c222c2195896b2cd9530f63ae92 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:41:22,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731962542763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43748 deadline: 1731962542763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962542765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:41:22,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:22,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:22,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:22,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:22,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:22,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741966_1142 (size=12104) 2024-11-18T20:41:22,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741967_1143 (size=14594) 2024-11-18T20:41:22,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962542867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731962542877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43748 deadline: 1731962542877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962542877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:22,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:41:22,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:22,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:22,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:22,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:22,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:41:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962543088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:23,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:23,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731962543089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:23,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43748 deadline: 1731962543089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:23,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962543088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:23,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:23,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:41:23,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:23,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:23,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:23,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,239 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:23,249 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/54193b2804d74d2b8964b52382458087 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/54193b2804d74d2b8964b52382458087 2024-11-18T20:41:23,253 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189a431d7515e74fd1b2b51c2cff0ce237_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189a431d7515e74fd1b2b51c2cff0ce237_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:23,256 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/08b3926ae5fc4cdc8d186019c0677437, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:23,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/08b3926ae5fc4cdc8d186019c0677437 is 175, key is test_row_0/A:col10/1731962482729/Put/seqid=0 2024-11-18T20:41:23,272 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/C of 59378c222c2195896b2cd9530f63ae92 into 54193b2804d74d2b8964b52382458087(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:23,272 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:23,272 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/C, priority=13, startTime=1731962482622; duration=0sec 2024-11-18T20:41:23,272 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:23,272 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:C 2024-11-18T20:41:23,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741968_1144 (size=39549) 2024-11-18T20:41:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:41:23,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:41:23,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:23,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:23,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:23,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962543408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:23,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962543408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731962543408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:23,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43748 deadline: 1731962543409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:23,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:41:23,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:23,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:23,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:23,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:41:23,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:23,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:23,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:23,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,641 INFO [master/c0a89b2656d4:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-18T20:41:23,641 INFO [master/c0a89b2656d4:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-18T20:41:23,686 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/08b3926ae5fc4cdc8d186019c0677437 2024-11-18T20:41:23,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/882e776eb9284b148a29faa13e4a9532 is 50, key is test_row_0/B:col10/1731962482729/Put/seqid=0 2024-11-18T20:41:23,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741969_1145 (size=12001) 2024-11-18T20:41:23,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:41:23,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:23,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:23,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:23,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:41:23,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:41:23,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:23,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:23,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:23,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:23,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962543918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:23,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731962543927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:23,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962543928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:23,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43748 deadline: 1731962543928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:23,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:23,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:24,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:41:24,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:24,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:24,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:24,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:24,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:24,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/882e776eb9284b148a29faa13e4a9532 2024-11-18T20:41:24,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/14c84cf58c3b47c3a65ddf0aa326c835 is 50, key is test_row_0/C:col10/1731962482729/Put/seqid=0 2024-11-18T20:41:24,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741970_1146 (size=12001) 2024-11-18T20:41:24,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/14c84cf58c3b47c3a65ddf0aa326c835 2024-11-18T20:41:24,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/08b3926ae5fc4cdc8d186019c0677437 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/08b3926ae5fc4cdc8d186019c0677437 2024-11-18T20:41:24,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/08b3926ae5fc4cdc8d186019c0677437, entries=200, sequenceid=98, filesize=38.6 K 2024-11-18T20:41:24,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/882e776eb9284b148a29faa13e4a9532 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/882e776eb9284b148a29faa13e4a9532 2024-11-18T20:41:24,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/882e776eb9284b148a29faa13e4a9532, entries=150, sequenceid=98, filesize=11.7 K 2024-11-18T20:41:24,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/14c84cf58c3b47c3a65ddf0aa326c835 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/14c84cf58c3b47c3a65ddf0aa326c835 2024-11-18T20:41:24,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/14c84cf58c3b47c3a65ddf0aa326c835, entries=150, sequenceid=98, filesize=11.7 K 2024-11-18T20:41:24,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 59378c222c2195896b2cd9530f63ae92 in 1439ms, sequenceid=98, compaction requested=false 2024-11-18T20:41:24,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:24,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:41:24,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:24,225 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T20:41:24,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:24,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:24,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:24,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:24,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:24,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:24,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111869d06872b3844392a9d27c0e4fe1abb0_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962482745/Put/seqid=0 2024-11-18T20:41:24,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741971_1147 (size=12154) 2024-11-18T20:41:24,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:24,272 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111869d06872b3844392a9d27c0e4fe1abb0_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111869d06872b3844392a9d27c0e4fe1abb0_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:24,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/01425a5ab96c4827a1724e84e44f6f55, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:24,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/01425a5ab96c4827a1724e84e44f6f55 is 175, key is test_row_0/A:col10/1731962482745/Put/seqid=0 2024-11-18T20:41:24,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741972_1148 (size=30955) 2024-11-18T20:41:24,296 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=111, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/01425a5ab96c4827a1724e84e44f6f55 2024-11-18T20:41:24,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/a4992e163d514978a4c34a2a8038ecf3 is 50, key is test_row_0/B:col10/1731962482745/Put/seqid=0 2024-11-18T20:41:24,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741973_1149 (size=12001) 2024-11-18T20:41:24,743 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/a4992e163d514978a4c34a2a8038ecf3 2024-11-18T20:41:24,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/b9d183d4e8894d95a8f6de577c75eddf is 50, key is test_row_0/C:col10/1731962482745/Put/seqid=0 2024-11-18T20:41:24,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741974_1150 (size=12001) 2024-11-18T20:41:24,778 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/b9d183d4e8894d95a8f6de577c75eddf 2024-11-18T20:41:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:24,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:24,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/01425a5ab96c4827a1724e84e44f6f55 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/01425a5ab96c4827a1724e84e44f6f55 2024-11-18T20:41:24,805 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/01425a5ab96c4827a1724e84e44f6f55, entries=150, sequenceid=111, filesize=30.2 K 2024-11-18T20:41:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:41:24,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/a4992e163d514978a4c34a2a8038ecf3 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/a4992e163d514978a4c34a2a8038ecf3 2024-11-18T20:41:24,824 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/a4992e163d514978a4c34a2a8038ecf3, entries=150, sequenceid=111, filesize=11.7 K 2024-11-18T20:41:24,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/b9d183d4e8894d95a8f6de577c75eddf as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/b9d183d4e8894d95a8f6de577c75eddf 2024-11-18T20:41:24,840 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/b9d183d4e8894d95a8f6de577c75eddf, entries=150, sequenceid=111, filesize=11.7 K 2024-11-18T20:41:24,844 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=60.38 KB/61830 for 59378c222c2195896b2cd9530f63ae92 in 619ms, sequenceid=111, compaction requested=true 2024-11-18T20:41:24,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:24,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:24,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-18T20:41:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-18T20:41:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:24,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T20:41:24,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-18T20:41:24,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:24,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1850 sec 2024-11-18T20:41:24,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:24,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:24,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:24,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:24,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:24,853 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.2020 sec 2024-11-18T20:41:24,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b3bb485c857f408fb798b63ea6cc3258_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962484848/Put/seqid=0 2024-11-18T20:41:24,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741975_1151 (size=17034) 2024-11-18T20:41:24,907 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:24,920 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b3bb485c857f408fb798b63ea6cc3258_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b3bb485c857f408fb798b63ea6cc3258_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:24,926 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/5056d12225e54da9addc3e5480586be3, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:24,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/5056d12225e54da9addc3e5480586be3 is 175, key is test_row_0/A:col10/1731962484848/Put/seqid=0 2024-11-18T20:41:24,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741976_1152 (size=48139) 2024-11-18T20:41:24,949 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=124, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/5056d12225e54da9addc3e5480586be3 2024-11-18T20:41:24,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731962544948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:24,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43748 deadline: 1731962544948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:24,957 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 59378c222c2195896b2cd9530f63ae92 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:24,958 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in 59378c222c2195896b2cd9530f63ae92 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:24,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962544948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:24,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962544953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:24,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:24,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:24,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:24,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962544957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:24,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/5497a1e1b95f48478c3127bcf5c96cf2 is 50, key is test_row_0/B:col10/1731962484848/Put/seqid=0 2024-11-18T20:41:25,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741977_1153 (size=12001) 2024-11-18T20:41:25,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/5497a1e1b95f48478c3127bcf5c96cf2 2024-11-18T20:41:25,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/5ec232c7d4544a4b80dbe546f586591a is 50, key is test_row_0/C:col10/1731962484848/Put/seqid=0 2024-11-18T20:41:25,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741978_1154 (size=12001) 2024-11-18T20:41:25,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/5ec232c7d4544a4b80dbe546f586591a 2024-11-18T20:41:25,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/5056d12225e54da9addc3e5480586be3 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/5056d12225e54da9addc3e5480586be3 2024-11-18T20:41:25,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962545069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:25,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962545069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:25,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/5056d12225e54da9addc3e5480586be3, entries=250, sequenceid=124, filesize=47.0 K 2024-11-18T20:41:25,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/5497a1e1b95f48478c3127bcf5c96cf2 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5497a1e1b95f48478c3127bcf5c96cf2 2024-11-18T20:41:25,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962545077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:25,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5497a1e1b95f48478c3127bcf5c96cf2, entries=150, sequenceid=124, filesize=11.7 K 2024-11-18T20:41:25,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/5ec232c7d4544a4b80dbe546f586591a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/5ec232c7d4544a4b80dbe546f586591a 2024-11-18T20:41:25,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/5ec232c7d4544a4b80dbe546f586591a, entries=150, sequenceid=124, filesize=11.7 K 2024-11-18T20:41:25,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 59378c222c2195896b2cd9530f63ae92 in 256ms, sequenceid=124, compaction requested=true 2024-11-18T20:41:25,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:25,106 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:41:25,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:41:25,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:25,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:41:25,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:25,107 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:41:25,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:41:25,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:25,107 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149701 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:41:25,107 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/A is initiating minor compaction (all files) 2024-11-18T20:41:25,108 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/A in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:25,108 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/d32f35e5980f47ec88b01cb26462cbab, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/08b3926ae5fc4cdc8d186019c0677437, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/01425a5ab96c4827a1724e84e44f6f55, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/5056d12225e54da9addc3e5480586be3] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=146.2 K 2024-11-18T20:41:25,108 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:25,108 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/d32f35e5980f47ec88b01cb26462cbab, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/08b3926ae5fc4cdc8d186019c0677437, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/01425a5ab96c4827a1724e84e44f6f55, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/5056d12225e54da9addc3e5480586be3] 2024-11-18T20:41:25,108 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:41:25,108 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/B is initiating minor compaction (all files) 2024-11-18T20:41:25,108 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/B in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:25,108 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/4647cae4ae6b46a491138a6087d759ae, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/882e776eb9284b148a29faa13e4a9532, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/a4992e163d514978a4c34a2a8038ecf3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5497a1e1b95f48478c3127bcf5c96cf2] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=47.0 K 2024-11-18T20:41:25,109 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting d32f35e5980f47ec88b01cb26462cbab, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731962481326 2024-11-18T20:41:25,109 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4647cae4ae6b46a491138a6087d759ae, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731962481326 2024-11-18T20:41:25,110 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08b3926ae5fc4cdc8d186019c0677437, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731962482087 2024-11-18T20:41:25,110 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 882e776eb9284b148a29faa13e4a9532, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731962482087 2024-11-18T20:41:25,110 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01425a5ab96c4827a1724e84e44f6f55, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731962482745 2024-11-18T20:41:25,110 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting a4992e163d514978a4c34a2a8038ecf3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731962482745 2024-11-18T20:41:25,111 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5056d12225e54da9addc3e5480586be3, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731962484784 2024-11-18T20:41:25,111 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 5497a1e1b95f48478c3127bcf5c96cf2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731962484784 2024-11-18T20:41:25,137 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:25,158 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#B#compaction#133 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:25,159 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/696a4a535c184532ad70f2ad59a65006 is 50, key is test_row_0/B:col10/1731962484848/Put/seqid=0 2024-11-18T20:41:25,166 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118e860ff10dcdf45c086524b5c775533d7_59378c222c2195896b2cd9530f63ae92 store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:25,171 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118e860ff10dcdf45c086524b5c775533d7_59378c222c2195896b2cd9530f63ae92, store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:25,171 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e860ff10dcdf45c086524b5c775533d7_59378c222c2195896b2cd9530f63ae92 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:25,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741979_1155 (size=12241) 2024-11-18T20:41:25,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741980_1156 (size=4469) 2024-11-18T20:41:25,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-18T20:41:25,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:25,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962484944, max timestamp=1731962485282Num uniques 859; 2024-11-18T20:41:25,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:25,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:25,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:41:25,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:25,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:25,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:25,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:25,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186e9471de7a9e44269bd85fc591e7c2b2_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962484944/Put/seqid=0 2024-11-18T20:41:25,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:25,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962545334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:25,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:25,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741981_1157 (size=12304) 2024-11-18T20:41:25,352 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:25,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962545347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:25,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962545348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:25,360 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186e9471de7a9e44269bd85fc591e7c2b2_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186e9471de7a9e44269bd85fc591e7c2b2_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:25,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:25,362 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/15839cad34684b379deb1e4d299bf0b6, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:25,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/15839cad34684b379deb1e4d299bf0b6 is 175, key is test_row_0/A:col10/1731962484944/Put/seqid=0 2024-11-18T20:41:25,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741982_1158 (size=31105) 2024-11-18T20:41:25,409 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/15839cad34684b379deb1e4d299bf0b6 2024-11-18T20:41:25,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/5f2d58342095491ab5ab270c2a323970 is 50, key is test_row_0/B:col10/1731962484944/Put/seqid=0 2024-11-18T20:41:25,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962545458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:25,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741983_1159 (size=12151) 2024-11-18T20:41:25,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962545467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:25,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962545468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:25,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:25,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:25,590 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/696a4a535c184532ad70f2ad59a65006 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/696a4a535c184532ad70f2ad59a65006 2024-11-18T20:41:25,600 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/B of 59378c222c2195896b2cd9530f63ae92 into 696a4a535c184532ad70f2ad59a65006(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:25,600 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:25,600 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/B, priority=12, startTime=1731962485107; duration=0sec 2024-11-18T20:41:25,600 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:25,600 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:B 2024-11-18T20:41:25,600 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:41:25,603 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:41:25,603 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/C is initiating minor compaction (all files) 2024-11-18T20:41:25,603 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/C in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:25,603 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/54193b2804d74d2b8964b52382458087, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/14c84cf58c3b47c3a65ddf0aa326c835, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/b9d183d4e8894d95a8f6de577c75eddf, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/5ec232c7d4544a4b80dbe546f586591a] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=47.0 K 2024-11-18T20:41:25,604 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 54193b2804d74d2b8964b52382458087, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731962481326 2024-11-18T20:41:25,604 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 14c84cf58c3b47c3a65ddf0aa326c835, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731962482087 2024-11-18T20:41:25,606 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting b9d183d4e8894d95a8f6de577c75eddf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731962482745 2024-11-18T20:41:25,606 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ec232c7d4544a4b80dbe546f586591a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731962484784 2024-11-18T20:41:25,622 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#A#compaction#132 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:25,624 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/77c7f68245c14632a750606453858079 is 175, key is test_row_0/A:col10/1731962484848/Put/seqid=0 2024-11-18T20:41:25,624 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#C#compaction#136 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:25,624 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/8751fc3350e44ef38f145e74b94543a7 is 50, key is test_row_0/C:col10/1731962484848/Put/seqid=0 2024-11-18T20:41:25,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741985_1161 (size=12241) 2024-11-18T20:41:25,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741984_1160 (size=31195) 2024-11-18T20:41:25,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962545667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:25,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962545687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962545687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:25,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/5f2d58342095491ab5ab270c2a323970 2024-11-18T20:41:25,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/0ef092ffdea048af81b7a713ecd80428 is 50, key is test_row_0/C:col10/1731962484944/Put/seqid=0 2024-11-18T20:41:25,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741986_1162 (size=12151) 2024-11-18T20:41:25,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962545988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:26,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962546007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:26,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962546007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:26,066 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/8751fc3350e44ef38f145e74b94543a7 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/8751fc3350e44ef38f145e74b94543a7 2024-11-18T20:41:26,075 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/C of 59378c222c2195896b2cd9530f63ae92 into 8751fc3350e44ef38f145e74b94543a7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:26,075 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:26,075 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/C, priority=12, startTime=1731962485107; duration=0sec 2024-11-18T20:41:26,075 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:26,075 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:C 2024-11-18T20:41:26,078 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/77c7f68245c14632a750606453858079 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/77c7f68245c14632a750606453858079 2024-11-18T20:41:26,088 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/A of 59378c222c2195896b2cd9530f63ae92 into 77c7f68245c14632a750606453858079(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:26,088 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:26,089 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/A, priority=12, startTime=1731962485106; duration=0sec 2024-11-18T20:41:26,089 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:26,089 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:A 2024-11-18T20:41:26,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/0ef092ffdea048af81b7a713ecd80428 2024-11-18T20:41:26,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/15839cad34684b379deb1e4d299bf0b6 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/15839cad34684b379deb1e4d299bf0b6 2024-11-18T20:41:26,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/15839cad34684b379deb1e4d299bf0b6, entries=150, sequenceid=145, filesize=30.4 K 2024-11-18T20:41:26,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/5f2d58342095491ab5ab270c2a323970 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5f2d58342095491ab5ab270c2a323970 2024-11-18T20:41:26,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5f2d58342095491ab5ab270c2a323970, entries=150, sequenceid=145, filesize=11.9 K 2024-11-18T20:41:26,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/0ef092ffdea048af81b7a713ecd80428 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/0ef092ffdea048af81b7a713ecd80428 2024-11-18T20:41:26,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/0ef092ffdea048af81b7a713ecd80428, entries=150, sequenceid=145, filesize=11.9 K 2024-11-18T20:41:26,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~91.69 KB/93890, heapSize ~267.14 KB/273552, currentSize=87.22 KB/89310 for 59378c222c2195896b2cd9530f63ae92 in 1086ms, sequenceid=145, compaction requested=false 2024-11-18T20:41:26,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:26,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T20:41:26,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:26,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:26,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:26,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:26,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:26,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:26,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bd26715fa0d545fb87b4a7300fb08d4e_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962486508/Put/seqid=0 2024-11-18T20:41:26,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741987_1163 (size=14794) 2024-11-18T20:41:26,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962546542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:26,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:26,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962546550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:26,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:26,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962546551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:26,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:26,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:26,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962546658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:26,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:26,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:26,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962546667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962546667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:41:26,828 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:26,828 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:41:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-18T20:41:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T20:41:26,830 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:41:26,831 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:41:26,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:41:26,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962546867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:26,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:26,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962546877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:26,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962546877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:26,935 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T20:41:26,941 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bd26715fa0d545fb87b4a7300fb08d4e_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bd26715fa0d545fb87b4a7300fb08d4e_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:26,943 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/9e035c9f99284f37b1e356e2723e3c3c, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:26,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/9e035c9f99284f37b1e356e2723e3c3c is 175, key is test_row_0/A:col10/1731962486508/Put/seqid=0 2024-11-18T20:41:26,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741988_1164 (size=39749) 2024-11-18T20:41:26,952 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/9e035c9f99284f37b1e356e2723e3c3c 2024-11-18T20:41:26,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/4a4a6121f96e4e8b964acde6b65cdf29 is 50, key is test_row_0/B:col10/1731962486508/Put/seqid=0 2024-11-18T20:41:26,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741989_1165 (size=12151) 2024-11-18T20:41:26,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731962546967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:26,976 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in 59378c222c2195896b2cd9530f63ae92 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:26,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43748 deadline: 1731962546977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:26,978 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 59378c222c2195896b2cd9530f63ae92 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:26,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:26,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:26,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T20:41:26,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:26,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:26,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:26,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:26,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:27,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T20:41:27,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:27,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:27,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:27,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:27,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T20:41:27,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962547187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:27,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962547188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:27,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962547189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:27,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T20:41:27,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:27,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:27,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:27,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:27,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:27,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/4a4a6121f96e4e8b964acde6b65cdf29 2024-11-18T20:41:27,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/f777375e05b346079c3adea3d182443e is 50, key is test_row_0/C:col10/1731962486508/Put/seqid=0 2024-11-18T20:41:27,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741990_1166 (size=12151) 2024-11-18T20:41:27,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/f777375e05b346079c3adea3d182443e 2024-11-18T20:41:27,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/9e035c9f99284f37b1e356e2723e3c3c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/9e035c9f99284f37b1e356e2723e3c3c 2024-11-18T20:41:27,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/9e035c9f99284f37b1e356e2723e3c3c, entries=200, sequenceid=165, filesize=38.8 K 2024-11-18T20:41:27,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/4a4a6121f96e4e8b964acde6b65cdf29 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/4a4a6121f96e4e8b964acde6b65cdf29 2024-11-18T20:41:27,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/4a4a6121f96e4e8b964acde6b65cdf29, entries=150, sequenceid=165, filesize=11.9 K 2024-11-18T20:41:27,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/f777375e05b346079c3adea3d182443e as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/f777375e05b346079c3adea3d182443e 2024-11-18T20:41:27,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/f777375e05b346079c3adea3d182443e, entries=150, sequenceid=165, filesize=11.9 K 2024-11-18T20:41:27,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 59378c222c2195896b2cd9530f63ae92 in 931ms, sequenceid=165, compaction requested=true 2024-11-18T20:41:27,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:27,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:41:27,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:27,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:41:27,441 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:27,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:27,442 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:27,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:41:27,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:27,444 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:27,444 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/B is initiating minor compaction (all files) 2024-11-18T20:41:27,444 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/B in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:27,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T20:41:27,444 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/696a4a535c184532ad70f2ad59a65006, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5f2d58342095491ab5ab270c2a323970, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/4a4a6121f96e4e8b964acde6b65cdf29] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=35.7 K 2024-11-18T20:41:27,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:27,444 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T20:41:27,444 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:27,444 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/A is initiating minor compaction (all files) 2024-11-18T20:41:27,445 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/A in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:27,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:27,445 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/77c7f68245c14632a750606453858079, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/15839cad34684b379deb1e4d299bf0b6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/9e035c9f99284f37b1e356e2723e3c3c] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=99.7 K 2024-11-18T20:41:27,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:27,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:27,445 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:27,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:27,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:27,445 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/77c7f68245c14632a750606453858079, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/15839cad34684b379deb1e4d299bf0b6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/9e035c9f99284f37b1e356e2723e3c3c] 2024-11-18T20:41:27,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:27,445 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 696a4a535c184532ad70f2ad59a65006, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731962484784 2024-11-18T20:41:27,445 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77c7f68245c14632a750606453858079, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731962484784 2024-11-18T20:41:27,446 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f2d58342095491ab5ab270c2a323970, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731962484944 2024-11-18T20:41:27,446 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15839cad34684b379deb1e4d299bf0b6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731962484944 2024-11-18T20:41:27,446 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a4a6121f96e4e8b964acde6b65cdf29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731962485332 2024-11-18T20:41:27,447 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e035c9f99284f37b1e356e2723e3c3c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731962485332 2024-11-18T20:41:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T20:41:27,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189fb92e8c04db47488b3f7041610548c5_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962486547/Put/seqid=0 2024-11-18T20:41:27,467 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#B#compaction#142 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:27,468 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/249fb47620c745719830ce116fcdac1f is 50, key is test_row_0/B:col10/1731962486508/Put/seqid=0 2024-11-18T20:41:27,469 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:27,479 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118ff1b3caf875743239381da4f6cb6a2dd_59378c222c2195896b2cd9530f63ae92 store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:27,482 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118ff1b3caf875743239381da4f6cb6a2dd_59378c222c2195896b2cd9530f63ae92, store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:27,482 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ff1b3caf875743239381da4f6cb6a2dd_59378c222c2195896b2cd9530f63ae92 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:27,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741991_1167 (size=12304) 2024-11-18T20:41:27,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,505 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189fb92e8c04db47488b3f7041610548c5_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189fb92e8c04db47488b3f7041610548c5_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:27,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/ef3e9905e7e348efac83ecf845c13be9, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:27,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/ef3e9905e7e348efac83ecf845c13be9 is 175, key is test_row_0/A:col10/1731962486547/Put/seqid=0 2024-11-18T20:41:27,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741992_1168 (size=12493) 2024-11-18T20:41:27,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741993_1169 (size=4469) 2024-11-18T20:41:27,532 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/249fb47620c745719830ce116fcdac1f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/249fb47620c745719830ce116fcdac1f 2024-11-18T20:41:27,541 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/B of 59378c222c2195896b2cd9530f63ae92 into 249fb47620c745719830ce116fcdac1f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:27,541 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:27,541 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/B, priority=13, startTime=1731962487441; duration=0sec 2024-11-18T20:41:27,542 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:27,542 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:B 2024-11-18T20:41:27,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741994_1170 (size=31105) 2024-11-18T20:41:27,543 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:27,545 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:27,545 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/C is initiating minor compaction (all files) 2024-11-18T20:41:27,545 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/C in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:27,545 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/8751fc3350e44ef38f145e74b94543a7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/0ef092ffdea048af81b7a713ecd80428, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/f777375e05b346079c3adea3d182443e] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=35.7 K 2024-11-18T20:41:27,545 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=181, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/ef3e9905e7e348efac83ecf845c13be9 2024-11-18T20:41:27,546 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 8751fc3350e44ef38f145e74b94543a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731962484784 2024-11-18T20:41:27,546 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ef092ffdea048af81b7a713ecd80428, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731962484944 2024-11-18T20:41:27,547 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting f777375e05b346079c3adea3d182443e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731962485332 2024-11-18T20:41:27,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/08d3ae9f540f4c31ba6f5ccc85a3e33b is 50, key is test_row_0/B:col10/1731962486547/Put/seqid=0 2024-11-18T20:41:27,562 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#C#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:27,563 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/aa66ed1ab38a461a905cade49cf375cc is 50, key is test_row_0/C:col10/1731962486508/Put/seqid=0 2024-11-18T20:41:27,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741995_1171 (size=12151) 2024-11-18T20:41:27,582 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/08d3ae9f540f4c31ba6f5ccc85a3e33b 2024-11-18T20:41:27,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741996_1172 (size=12493) 2024-11-18T20:41:27,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/e2d5449ab1be4006b2ab6fcd22cf84f9 is 50, key is test_row_0/C:col10/1731962486547/Put/seqid=0 2024-11-18T20:41:27,601 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/aa66ed1ab38a461a905cade49cf375cc as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/aa66ed1ab38a461a905cade49cf375cc 2024-11-18T20:41:27,609 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/C of 59378c222c2195896b2cd9530f63ae92 into aa66ed1ab38a461a905cade49cf375cc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:27,609 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:27,609 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/C, priority=13, startTime=1731962487442; duration=0sec 2024-11-18T20:41:27,609 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:27,609 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:C 2024-11-18T20:41:27,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741997_1173 (size=12151) 2024-11-18T20:41:27,622 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/e2d5449ab1be4006b2ab6fcd22cf84f9 2024-11-18T20:41:27,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/ef3e9905e7e348efac83ecf845c13be9 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/ef3e9905e7e348efac83ecf845c13be9 2024-11-18T20:41:27,639 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/ef3e9905e7e348efac83ecf845c13be9, entries=150, sequenceid=181, filesize=30.4 K 2024-11-18T20:41:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/08d3ae9f540f4c31ba6f5ccc85a3e33b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/08d3ae9f540f4c31ba6f5ccc85a3e33b 2024-11-18T20:41:27,647 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/08d3ae9f540f4c31ba6f5ccc85a3e33b, entries=150, sequenceid=181, filesize=11.9 K 2024-11-18T20:41:27,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/e2d5449ab1be4006b2ab6fcd22cf84f9 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/e2d5449ab1be4006b2ab6fcd22cf84f9 2024-11-18T20:41:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,658 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/e2d5449ab1be4006b2ab6fcd22cf84f9, entries=150, sequenceid=181, filesize=11.9 K 2024-11-18T20:41:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,659 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 59378c222c2195896b2cd9530f63ae92 in 215ms, sequenceid=181, compaction requested=false 2024-11-18T20:41:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:27,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:27,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-18T20:41:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-18T20:41:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,663 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-18T20:41:27,663 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 830 msec 2024-11-18T20:41:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,666 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 836 msec 2024-11-18T20:41:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:27,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T20:41:27,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:27,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:27,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e9de93b5a40a4b6e80153adc747f7eec_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962487731/Put/seqid=0 2024-11-18T20:41:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741998_1174 (size=22268) 2024-11-18T20:41:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,811 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,817 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e9de93b5a40a4b6e80153adc747f7eec_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e9de93b5a40a4b6e80153adc747f7eec_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,818 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/fba1604ec236457799be9a2c20b6b147, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:27,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/fba1604ec236457799be9a2c20b6b147 is 175, key is test_row_0/A:col10/1731962487731/Put/seqid=0 2024-11-18T20:41:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731962487758, max timestamp=1731962487833Num uniques 859; 2024-11-18T20:41:27,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-18T20:41:27,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731962487758, max timestamp=1731962487833Num uniques 859; 2024-11-18T20:41:27,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-18T20:41:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731962487758, max timestamp=1731962487833Num uniques 859; 2024-11-18T20:41:27,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-18T20:41:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741999_1175 (size=65673) 2024-11-18T20:41:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,848 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/fba1604ec236457799be9a2c20b6b147 2024-11-18T20:41:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/482407928e044424a82f7e14ebeff7e7 is 50, key is test_row_0/B:col10/1731962487731/Put/seqid=0 2024-11-18T20:41:27,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962547850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:27,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962547850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:27,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962547852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:27,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:27,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:27,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742000_1176 (size=21715) 2024-11-18T20:41:27,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/482407928e044424a82f7e14ebeff7e7 2024-11-18T20:41:27,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/1ed1663a31b5458eb369e71a8cca1728 is 50, key is test_row_0/C:col10/1731962487731/Put/seqid=0 2024-11-18T20:41:27,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742001_1177 (size=12151) 2024-11-18T20:41:27,931 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#A#compaction#143 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:27,932 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/01382eff060547f1864cbbe807b29e9b is 175, key is test_row_0/A:col10/1731962486508/Put/seqid=0 2024-11-18T20:41:27,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742002_1178 (size=31447) 2024-11-18T20:41:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T20:41:27,967 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:27,967 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:41:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-18T20:41:27,970 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:41:27,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:41:27,970 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:41:27,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:41:27,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962547967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:27,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962547967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:27,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962547967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:41:28,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T20:41:28,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:28,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:28,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:28,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:28,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:28,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962548177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962548177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:28,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962548187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:28,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T20:41:28,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:28,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:28,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:28,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:28,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:41:28,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/1ed1663a31b5458eb369e71a8cca1728 2024-11-18T20:41:28,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/fba1604ec236457799be9a2c20b6b147 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/fba1604ec236457799be9a2c20b6b147 2024-11-18T20:41:28,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/fba1604ec236457799be9a2c20b6b147, entries=350, sequenceid=193, filesize=64.1 K 2024-11-18T20:41:28,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/482407928e044424a82f7e14ebeff7e7 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/482407928e044424a82f7e14ebeff7e7 2024-11-18T20:41:28,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/482407928e044424a82f7e14ebeff7e7, entries=350, sequenceid=193, filesize=21.2 K 2024-11-18T20:41:28,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/1ed1663a31b5458eb369e71a8cca1728 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/1ed1663a31b5458eb369e71a8cca1728 2024-11-18T20:41:28,354 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/01382eff060547f1864cbbe807b29e9b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/01382eff060547f1864cbbe807b29e9b 2024-11-18T20:41:28,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/1ed1663a31b5458eb369e71a8cca1728, entries=150, sequenceid=193, filesize=11.9 K 2024-11-18T20:41:28,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for 59378c222c2195896b2cd9530f63ae92 in 604ms, sequenceid=193, compaction requested=true 2024-11-18T20:41:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:41:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:41:28,359 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-18T20:41:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:41:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:41:28,361 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T20:41:28,361 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T20:41:28,362 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. because compaction request was cancelled 2024-11-18T20:41:28,362 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:A 2024-11-18T20:41:28,362 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:28,362 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/A of 59378c222c2195896b2cd9530f63ae92 into 01382eff060547f1864cbbe807b29e9b(size=30.7 K), total size for store is 125.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:28,362 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:28,362 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/A, priority=13, startTime=1731962487441; duration=0sec 2024-11-18T20:41:28,362 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:28,362 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:A 2024-11-18T20:41:28,362 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:28,363 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 46359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:28,364 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/B is initiating minor compaction (all files) 2024-11-18T20:41:28,364 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/B in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:28,364 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/249fb47620c745719830ce116fcdac1f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/08d3ae9f540f4c31ba6f5ccc85a3e33b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/482407928e044424a82f7e14ebeff7e7] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=45.3 K 2024-11-18T20:41:28,364 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:28,364 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 249fb47620c745719830ce116fcdac1f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731962485332 2024-11-18T20:41:28,364 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/C is initiating minor compaction (all files) 2024-11-18T20:41:28,364 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/C in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:28,364 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/aa66ed1ab38a461a905cade49cf375cc, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/e2d5449ab1be4006b2ab6fcd22cf84f9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/1ed1663a31b5458eb369e71a8cca1728] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=35.9 K 2024-11-18T20:41:28,365 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 08d3ae9f540f4c31ba6f5ccc85a3e33b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731962486540 2024-11-18T20:41:28,365 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa66ed1ab38a461a905cade49cf375cc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731962485332 2024-11-18T20:41:28,365 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 482407928e044424a82f7e14ebeff7e7, keycount=350, bloomtype=ROW, size=21.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731962487728 2024-11-18T20:41:28,365 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2d5449ab1be4006b2ab6fcd22cf84f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731962486540 2024-11-18T20:41:28,366 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ed1663a31b5458eb369e71a8cca1728, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731962487731 2024-11-18T20:41:28,373 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#B#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:28,373 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/b31a5c5b3faa41c0a7b000ef57798ca1 is 50, key is test_row_0/B:col10/1731962487731/Put/seqid=0 2024-11-18T20:41:28,376 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#C#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:28,377 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/cf14aa850e084555bce2c822a037561d is 50, key is test_row_0/C:col10/1731962487731/Put/seqid=0 2024-11-18T20:41:28,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742003_1179 (size=12595) 2024-11-18T20:41:28,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742004_1180 (size=12595) 2024-11-18T20:41:28,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T20:41:28,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:28,434 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=154.31 KB heapSize=379.13 KB 2024-11-18T20:41:28,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:28,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:28,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:28,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:28,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:28,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:28,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f787e2dcd68e466181e74b2ece1900e9_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962487847/Put/seqid=0 2024-11-18T20:41:28,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742005_1181 (size=12304) 2024-11-18T20:41:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:28,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:28,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962548504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962548505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:28,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962548507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:41:28,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962548617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:28,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:28,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962548617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:28,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:28,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962548618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:28,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:28,798 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/b31a5c5b3faa41c0a7b000ef57798ca1 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/b31a5c5b3faa41c0a7b000ef57798ca1 2024-11-18T20:41:28,806 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/B of 59378c222c2195896b2cd9530f63ae92 into b31a5c5b3faa41c0a7b000ef57798ca1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:28,807 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:28,807 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/B, priority=13, startTime=1731962488359; duration=0sec 2024-11-18T20:41:28,807 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:28,807 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:B 2024-11-18T20:41:28,816 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/cf14aa850e084555bce2c822a037561d as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/cf14aa850e084555bce2c822a037561d 2024-11-18T20:41:28,824 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/C of 59378c222c2195896b2cd9530f63ae92 into cf14aa850e084555bce2c822a037561d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:28,824 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:28,824 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/C, priority=13, startTime=1731962488359; duration=0sec 2024-11-18T20:41:28,825 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:28,825 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:C 2024-11-18T20:41:28,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962548827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:28,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962548827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:28,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962548827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:28,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:28,868 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f787e2dcd68e466181e74b2ece1900e9_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f787e2dcd68e466181e74b2ece1900e9_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:28,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/8ce627136e99499e9950820f3183f1c6, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:28,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/8ce627136e99499e9950820f3183f1c6 is 175, key is test_row_0/A:col10/1731962487847/Put/seqid=0 2024-11-18T20:41:28,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742006_1182 (size=31105) 2024-11-18T20:41:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:41:29,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962549138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:29,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962549138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:29,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962549147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:29,275 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/8ce627136e99499e9950820f3183f1c6 2024-11-18T20:41:29,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/201614f617e8470b8e6ec5309242d92c is 50, key is test_row_0/B:col10/1731962487847/Put/seqid=0 2024-11-18T20:41:29,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742007_1183 (size=12151) 2024-11-18T20:41:29,294 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/201614f617e8470b8e6ec5309242d92c 2024-11-18T20:41:29,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/7a3a081eeb574ca39e04b7486c6182e6 is 50, key is test_row_0/C:col10/1731962487847/Put/seqid=0 2024-11-18T20:41:29,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742008_1184 (size=12151) 2024-11-18T20:41:29,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962549647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:29,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962549647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:29,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962549667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:29,710 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/7a3a081eeb574ca39e04b7486c6182e6 2024-11-18T20:41:29,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/8ce627136e99499e9950820f3183f1c6 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/8ce627136e99499e9950820f3183f1c6 2024-11-18T20:41:29,722 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/8ce627136e99499e9950820f3183f1c6, entries=150, sequenceid=220, filesize=30.4 K 2024-11-18T20:41:29,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/201614f617e8470b8e6ec5309242d92c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/201614f617e8470b8e6ec5309242d92c 2024-11-18T20:41:29,729 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/201614f617e8470b8e6ec5309242d92c, entries=150, sequenceid=220, filesize=11.9 K 2024-11-18T20:41:29,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/7a3a081eeb574ca39e04b7486c6182e6 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/7a3a081eeb574ca39e04b7486c6182e6 2024-11-18T20:41:29,736 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/7a3a081eeb574ca39e04b7486c6182e6, entries=150, sequenceid=220, filesize=11.9 K 2024-11-18T20:41:29,737 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~379.08 KB/388176, currentSize=46.96 KB/48090 for 59378c222c2195896b2cd9530f63ae92 in 1303ms, sequenceid=220, compaction requested=true 2024-11-18T20:41:29,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:29,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:29,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-18T20:41:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-18T20:41:29,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-18T20:41:29,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7670 sec 2024-11-18T20:41:29,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.7740 sec 2024-11-18T20:41:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:41:30,128 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:30,128 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:41:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-18T20:41:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:41:30,131 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:41:30,132 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:41:30,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:41:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:41:30,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T20:41:30,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:30,286 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T20:41:30,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:30,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:30,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:30,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:30,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:30,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:30,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111800e8c2ee9d434808aaf29b8b49cdaa24_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962488504/Put/seqid=0 2024-11-18T20:41:30,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742009_1185 (size=12304) 2024-11-18T20:41:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:41:30,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:30,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:30,708 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111800e8c2ee9d434808aaf29b8b49cdaa24_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111800e8c2ee9d434808aaf29b8b49cdaa24_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:30,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/243ead474ab04f8f94b0b1eef72c51f7, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:30,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/243ead474ab04f8f94b0b1eef72c51f7 is 175, key is test_row_0/A:col10/1731962488504/Put/seqid=0 2024-11-18T20:41:30,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962490694, max timestamp=1731962490711Num uniques 859; 2024-11-18T20:41:30,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:41:30,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962490694, max timestamp=1731962490711Num uniques 859; 2024-11-18T20:41:30,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:41:30,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742010_1186 (size=31105) 2024-11-18T20:41:30,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962490694, max timestamp=1731962490711Num uniques 859; 2024-11-18T20:41:30,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:41:30,720 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/243ead474ab04f8f94b0b1eef72c51f7 2024-11-18T20:41:30,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/b40d5605a4a448b6969d4072cd3cbf04 is 50, key is test_row_0/B:col10/1731962488504/Put/seqid=0 2024-11-18T20:41:30,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742011_1187 (size=12151) 2024-11-18T20:41:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:41:30,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962550752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:30,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962550753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:30,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:30,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962550763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:30,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:30,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:30,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:30,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962550867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:30,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962550868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:30,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962550877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:30,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:30,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:30,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:31,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43748 deadline: 1731962550997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:31,005 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 59378c222c2195896b2cd9530f63ae92 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:31,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:31,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:31,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:31,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731962551018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:31,022 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in 59378c222c2195896b2cd9530f63ae92 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:31,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962551077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:31,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962551077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:31,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962551087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:31,152 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/b40d5605a4a448b6969d4072cd3cbf04 2024-11-18T20:41:31,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/eae20893234c4468a9b3f8cc98252f8f is 50, key is test_row_0/C:col10/1731962488504/Put/seqid=0 2024-11-18T20:41:31,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742012_1188 (size=12151) 2024-11-18T20:41:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:41:31,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962551388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:31,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:31,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:31,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:31,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962551389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:31,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:31,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962551397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:31,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:31,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:31,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:31,575 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/eae20893234c4468a9b3f8cc98252f8f 2024-11-18T20:41:31,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/243ead474ab04f8f94b0b1eef72c51f7 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/243ead474ab04f8f94b0b1eef72c51f7 2024-11-18T20:41:31,585 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/243ead474ab04f8f94b0b1eef72c51f7, entries=150, sequenceid=232, filesize=30.4 K 2024-11-18T20:41:31,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/b40d5605a4a448b6969d4072cd3cbf04 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/b40d5605a4a448b6969d4072cd3cbf04 2024-11-18T20:41:31,593 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/b40d5605a4a448b6969d4072cd3cbf04, entries=150, sequenceid=232, filesize=11.9 K 2024-11-18T20:41:31,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/eae20893234c4468a9b3f8cc98252f8f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/eae20893234c4468a9b3f8cc98252f8f 2024-11-18T20:41:31,598 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/eae20893234c4468a9b3f8cc98252f8f, entries=150, sequenceid=232, filesize=11.9 K 2024-11-18T20:41:31,599 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 59378c222c2195896b2cd9530f63ae92 in 1313ms, sequenceid=232, compaction requested=true 2024-11-18T20:41:31,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:31,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:31,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-18T20:41:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-18T20:41:31,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-18T20:41:31,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4690 sec 2024-11-18T20:41:31,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4750 sec 2024-11-18T20:41:31,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:31,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731962490740, max timestamp=1731962491900Num uniques 1018; 2024-11-18T20:41:31,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T20:41:31,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731962490740, max timestamp=1731962491900Num uniques 1018; 2024-11-18T20:41:31,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731962490740, max timestamp=1731962491900Num uniques 1018; 2024-11-18T20:41:31,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T20:41:31,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T20:41:31,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=26.84 KB heapSize=65.39 KB 2024-11-18T20:41:31,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:31,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:31,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:31,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:31,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:31,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:31,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187713ed11880547848ebceaf3b4d51456_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962490740/Put/seqid=0 2024-11-18T20:41:31,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742013_1189 (size=12454) 2024-11-18T20:41:31,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962491947, max timestamp=1731962491958Num uniques 859; 2024-11-18T20:41:31,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:41:31,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962491947, max timestamp=1731962491958Num uniques 859; 2024-11-18T20:41:31,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:41:31,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962491947, max timestamp=1731962491958Num uniques 859; 2024-11-18T20:41:31,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:41:32,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731962491984, max timestamp=1731962492001Num uniques 1068; 2024-11-18T20:41:32,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731962491984, max timestamp=1731962492001Num uniques 1068; 2024-11-18T20:41:32,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-18T20:41:32,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-18T20:41:32,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731962491984, max timestamp=1731962492001Num uniques 1068; 2024-11-18T20:41:32,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-18T20:41:32,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962552055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:32,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:32,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962552056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:32,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731962492047, max timestamp=1731962492060Num uniques 1177; 2024-11-18T20:41:32,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731962492047, max timestamp=1731962492060Num uniques 1177; 2024-11-18T20:41:32,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-11-18T20:41:32,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-11-18T20:41:32,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731962492047, max timestamp=1731962492060Num uniques 1177; 2024-11-18T20:41:32,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-11-18T20:41:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:32,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731962492175, max timestamp=1731962492192Num uniques 1036; 2024-11-18T20:41:32,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731962492175, max timestamp=1731962492192Num uniques 1036; 2024-11-18T20:41:32,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731962492175, max timestamp=1731962492192Num uniques 1036; 2024-11-18T20:41:32,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-18T20:41:32,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-18T20:41:32,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-18T20:41:32,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731962492216, max timestamp=1731962492247Num uniques 1045; 2024-11-18T20:41:32,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731962492216, max timestamp=1731962492247Num uniques 1045; 2024-11-18T20:41:32,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-18T20:41:32,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731962492216, max timestamp=1731962492247Num uniques 1045; 2024-11-18T20:41:32,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-18T20:41:32,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-18T20:41:32,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:41:32,278 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:32,278 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:41:32,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-18T20:41:32,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T20:41:32,281 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:41:32,284 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:41:32,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:41:32,322 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:32,327 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187713ed11880547848ebceaf3b4d51456_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187713ed11880547848ebceaf3b4d51456_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:32,328 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/93a7fb41187e420e9eed1fbc420c6b09, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:32,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/93a7fb41187e420e9eed1fbc420c6b09 is 175, key is test_row_0/A:col10/1731962490740/Put/seqid=0 2024-11-18T20:41:32,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742014_1190 (size=31255) 2024-11-18T20:41:32,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:32,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962552328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:32,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:32,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:32,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:32,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962552329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:32,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1731962492306, max timestamp=1731962492334Num uniques 1154; 2024-11-18T20:41:32,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=13920, suffix heap size=147984,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=304 2024-11-18T20:41:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:32,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1731962492306, max timestamp=1731962492334Num uniques 1154; 2024-11-18T20:41:32,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=13920, suffix heap size=147984,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=304 2024-11-18T20:41:32,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1731962492306, max timestamp=1731962492334Num uniques 1154; 2024-11-18T20:41:32,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=13920, suffix heap size=147984,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=304 2024-11-18T20:41:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T20:41:32,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-18T20:41:32,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:32,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:32,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:32,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:32,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:32,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:32,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:32,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962552451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:32,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:32,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962552452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:32,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:32,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:32,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:32,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:32,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:32,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:32,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=363, cellSize=16618, totalHeapSize=37992, min timestamp=1731962492379, max timestamp=1731962492460Num uniques 1163; 2024-11-18T20:41:32,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=16618, suffix heap size=148920,new segment heap size=37992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=363 2024-11-18T20:41:32,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=363, cellSize=16618, totalHeapSize=37992, min timestamp=1731962492379, max timestamp=1731962492460Num uniques 1163; 2024-11-18T20:41:32,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=363, cellSize=16618, totalHeapSize=37992, min timestamp=1731962492379, max timestamp=1731962492460Num uniques 1163; 2024-11-18T20:41:32,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=16618, suffix heap size=148920,new segment heap size=37992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=363 2024-11-18T20:41:32,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=16618, suffix heap size=148920,new segment heap size=37992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=363 2024-11-18T20:41:32,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-18T20:41:32,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:32,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:32,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:32,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:32,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:32,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T20:41:32,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962552613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:32,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962552613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:32,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:32,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:32,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:32,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:32,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:32,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:32,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731962492554, max timestamp=1731962492626Num uniques 1222; 2024-11-18T20:41:32,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=55962, new segment data size=7872, suffix heap size=155056,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1222, new segment cells count=172 2024-11-18T20:41:32,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731962492554, max timestamp=1731962492626Num uniques 1222; 2024-11-18T20:41:32,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=55962, new segment data size=7872, suffix heap size=155056,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1222, new segment cells count=172 2024-11-18T20:41:32,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731962492554, max timestamp=1731962492626Num uniques 1222; 2024-11-18T20:41:32,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=55962, new segment data size=7872, suffix heap size=155056,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1222, new segment cells count=172 2024-11-18T20:41:32,735 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/93a7fb41187e420e9eed1fbc420c6b09 2024-11-18T20:41:32,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-18T20:41:32,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:32,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:32,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:32,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:32,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:32,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:32,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731962492653, max timestamp=1731962492756Num uniques 1031; 2024-11-18T20:41:32,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=12866, suffix heap size=135192,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=281 2024-11-18T20:41:32,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731962492653, max timestamp=1731962492756Num uniques 1031; 2024-11-18T20:41:32,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=12866, suffix heap size=135192,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=281 2024-11-18T20:41:32,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731962492653, max timestamp=1731962492756Num uniques 1031; 2024-11-18T20:41:32,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=12866, suffix heap size=135192,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=281 2024-11-18T20:41:32,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/5c2f59c9aff84a41b18dc17f8d97cfdb is 50, key is test_row_0/B:col10/1731962490740/Put/seqid=0 2024-11-18T20:41:32,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731962492758, max timestamp=1731962492819Num uniques 1140; 2024-11-18T20:41:32,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-11-18T20:41:32,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731962492758, max timestamp=1731962492819Num uniques 1140; 2024-11-18T20:41:32,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-11-18T20:41:32,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731962492758, max timestamp=1731962492819Num uniques 1140; 2024-11-18T20:41:32,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-11-18T20:41:32,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742015_1191 (size=12301) 2024-11-18T20:41:32,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1731962492875, max timestamp=1731962492890Num uniques 1099; 2024-11-18T20:41:32,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1731962492875, max timestamp=1731962492890Num uniques 1099; 2024-11-18T20:41:32,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50324, new segment data size=9104, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-11-18T20:41:32,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50324, new segment data size=9104, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-11-18T20:41:32,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1731962492875, max timestamp=1731962492890Num uniques 1099; 2024-11-18T20:41:32,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50324, new segment data size=9104, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-11-18T20:41:32,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-18T20:41:32,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:32,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:32,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:32,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:32,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T20:41:32,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=358, cellSize=16390, totalHeapSize=37472, min timestamp=1731962492914, max timestamp=1731962492957Num uniques 1058; 2024-11-18T20:41:32,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=16390, suffix heap size=138000,new segment heap size=37472 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=358 2024-11-18T20:41:32,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=358, cellSize=16390, totalHeapSize=37472, min timestamp=1731962492914, max timestamp=1731962492957Num uniques 1058; 2024-11-18T20:41:32,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=16390, suffix heap size=138000,new segment heap size=37472 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=358 2024-11-18T20:41:32,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=358, cellSize=16390, totalHeapSize=37472, min timestamp=1731962492914, max timestamp=1731962492957Num uniques 1058; 2024-11-18T20:41:32,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=16390, suffix heap size=138000,new segment heap size=37472 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=358 2024-11-18T20:41:33,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962552992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:33,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=367, cellSize=16802, totalHeapSize=38408, min timestamp=1731962492966, max timestamp=1731962493004Num uniques 1217; 2024-11-18T20:41:33,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=367, cellSize=16802, totalHeapSize=38408, min timestamp=1731962492966, max timestamp=1731962493004Num uniques 1217; 2024-11-18T20:41:33,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=55732, new segment data size=16802, suffix heap size=154536,new segment heap size=38408 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1217, new segment cells count=367 2024-11-18T20:41:33,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=367, cellSize=16802, totalHeapSize=38408, min timestamp=1731962492966, max timestamp=1731962493004Num uniques 1217; 2024-11-18T20:41:33,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=55732, new segment data size=16802, suffix heap size=154536,new segment heap size=38408 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1217, new segment cells count=367 2024-11-18T20:41:33,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=55732, new segment data size=16802, suffix heap size=154536,new segment heap size=38408 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1217, new segment cells count=367 2024-11-18T20:41:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:33,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-18T20:41:33,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:33,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:33,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:33,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:33,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:33,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731962493079, max timestamp=1731962493098Num uniques 1226; 2024-11-18T20:41:33,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731962493079, max timestamp=1731962493098Num uniques 1226; 2024-11-18T20:41:33,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:33,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=56146, new segment data size=10346, suffix heap size=155472,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1226, new segment cells count=226 2024-11-18T20:41:33,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=56146, new segment data size=10346, suffix heap size=155472,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1226, new segment cells count=226 2024-11-18T20:41:33,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962553102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:33,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731962493079, max timestamp=1731962493098Num uniques 1226; 2024-11-18T20:41:33,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=56146, new segment data size=10346, suffix heap size=155472,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1226, new segment cells count=226 2024-11-18T20:41:33,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:33,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731962493163, max timestamp=1731962493178Num uniques 1085; 2024-11-18T20:41:33,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=10760, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-11-18T20:41:33,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731962493163, max timestamp=1731962493178Num uniques 1085; 2024-11-18T20:41:33,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=10760, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-11-18T20:41:33,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731962493163, max timestamp=1731962493178Num uniques 1085; 2024-11-18T20:41:33,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=10760, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-11-18T20:41:33,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-18T20:41:33,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:33,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:33,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:33,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:33,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:33,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=444, cellSize=20334, totalHeapSize=46416, min timestamp=1731962493200, max timestamp=1731962493241Num uniques 1094; 2024-11-18T20:41:33,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=20334, suffix heap size=141744,new segment heap size=46416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=444 2024-11-18T20:41:33,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=444, cellSize=20334, totalHeapSize=46416, min timestamp=1731962493200, max timestamp=1731962493241Num uniques 1094; 2024-11-18T20:41:33,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=20334, suffix heap size=141744,new segment heap size=46416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=444 2024-11-18T20:41:33,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/5c2f59c9aff84a41b18dc17f8d97cfdb 2024-11-18T20:41:33,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=444, cellSize=20334, totalHeapSize=46416, min timestamp=1731962493200, max timestamp=1731962493241Num uniques 1094; 2024-11-18T20:41:33,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=20334, suffix heap size=141744,new segment heap size=46416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=444 2024-11-18T20:41:33,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/c9ba20a8e8a84cb9807c4668b6c7d9df is 50, key is test_row_0/C:col10/1731962490740/Put/seqid=0 2024-11-18T20:41:33,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742016_1192 (size=12301) 2024-11-18T20:41:33,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/c9ba20a8e8a84cb9807c4668b6c7d9df 2024-11-18T20:41:33,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/93a7fb41187e420e9eed1fbc420c6b09 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/93a7fb41187e420e9eed1fbc420c6b09 2024-11-18T20:41:33,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/93a7fb41187e420e9eed1fbc420c6b09, entries=150, sequenceid=270, filesize=30.5 K 2024-11-18T20:41:33,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/5c2f59c9aff84a41b18dc17f8d97cfdb as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5c2f59c9aff84a41b18dc17f8d97cfdb 2024-11-18T20:41:33,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:33,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962553298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:33,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:33,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962553300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:33,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:33,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:33,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962553309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:33,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:33,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:33,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5c2f59c9aff84a41b18dc17f8d97cfdb, entries=150, sequenceid=270, filesize=12.0 K 2024-11-18T20:41:33,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/c9ba20a8e8a84cb9807c4668b6c7d9df as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/c9ba20a8e8a84cb9807c4668b6c7d9df 2024-11-18T20:41:33,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/c9ba20a8e8a84cb9807c4668b6c7d9df, entries=150, sequenceid=270, filesize=12.0 K 2024-11-18T20:41:33,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=167.72 KB/171750 for 59378c222c2195896b2cd9530f63ae92 in 1441ms, sequenceid=270, compaction requested=true 2024-11-18T20:41:33,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:33,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:41:33,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:33,347 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-18T20:41:33,347 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:41:33,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:41:33,348 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:41:33,348 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/B is initiating minor compaction (all files) 2024-11-18T20:41:33,348 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/B in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:33,349 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/b31a5c5b3faa41c0a7b000ef57798ca1, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/201614f617e8470b8e6ec5309242d92c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/b40d5605a4a448b6969d4072cd3cbf04, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5c2f59c9aff84a41b18dc17f8d97cfdb] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=48.0 K 2024-11-18T20:41:33,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:33,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:41:33,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:33,349 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting b31a5c5b3faa41c0a7b000ef57798ca1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731962487731 2024-11-18T20:41:33,349 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 221690 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-18T20:41:33,350 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/A is initiating minor compaction (all files) 2024-11-18T20:41:33,350 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/A in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:33,350 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 201614f617e8470b8e6ec5309242d92c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731962487846 2024-11-18T20:41:33,350 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/01382eff060547f1864cbbe807b29e9b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/ef3e9905e7e348efac83ecf845c13be9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/fba1604ec236457799be9a2c20b6b147, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/8ce627136e99499e9950820f3183f1c6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/243ead474ab04f8f94b0b1eef72c51f7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/93a7fb41187e420e9eed1fbc420c6b09] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=216.5 K 2024-11-18T20:41:33,350 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:33,350 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/01382eff060547f1864cbbe807b29e9b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/ef3e9905e7e348efac83ecf845c13be9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/fba1604ec236457799be9a2c20b6b147, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/8ce627136e99499e9950820f3183f1c6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/243ead474ab04f8f94b0b1eef72c51f7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/93a7fb41187e420e9eed1fbc420c6b09] 2024-11-18T20:41:33,350 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting b40d5605a4a448b6969d4072cd3cbf04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731962488490 2024-11-18T20:41:33,350 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01382eff060547f1864cbbe807b29e9b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731962485332 2024-11-18T20:41:33,352 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c2f59c9aff84a41b18dc17f8d97cfdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731962490740 2024-11-18T20:41:33,352 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef3e9905e7e348efac83ecf845c13be9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731962486540 2024-11-18T20:41:33,352 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting fba1604ec236457799be9a2c20b6b147, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731962487728 2024-11-18T20:41:33,352 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ce627136e99499e9950820f3183f1c6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731962487846 2024-11-18T20:41:33,353 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 243ead474ab04f8f94b0b1eef72c51f7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731962488490 2024-11-18T20:41:33,353 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93a7fb41187e420e9eed1fbc420c6b09, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731962490740 2024-11-18T20:41:33,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-18T20:41:33,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:33,361 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=167.72 KB heapSize=457.88 KB 2024-11-18T20:41:33,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:33,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:33,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:33,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:33,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:33,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:33,370 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#B#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:33,371 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/2d8d8dfe2810420d83fe9e38cd2b1313 is 50, key is test_row_0/B:col10/1731962490740/Put/seqid=0 2024-11-18T20:41:33,383 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:33,390 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118b9b5ffbdd6a14c6389dac3869e92b401_59378c222c2195896b2cd9530f63ae92 store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:33,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a35f0bb7923143469a95f183ac56a0aa_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962493296/Put/seqid=0 2024-11-18T20:41:33,405 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118b9b5ffbdd6a14c6389dac3869e92b401_59378c222c2195896b2cd9530f63ae92, store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:33,405 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b9b5ffbdd6a14c6389dac3869e92b401_59378c222c2195896b2cd9530f63ae92 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:33,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742017_1193 (size=12881) 2024-11-18T20:41:33,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T20:41:33,423 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/2d8d8dfe2810420d83fe9e38cd2b1313 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/2d8d8dfe2810420d83fe9e38cd2b1313 2024-11-18T20:41:33,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:33,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:33,435 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/B of 59378c222c2195896b2cd9530f63ae92 into 2d8d8dfe2810420d83fe9e38cd2b1313(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:33,435 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:33,435 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/B, priority=12, startTime=1731962493347; duration=0sec 2024-11-18T20:41:33,435 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:33,435 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:B 2024-11-18T20:41:33,435 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:41:33,438 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:41:33,439 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/C is initiating minor compaction (all files) 2024-11-18T20:41:33,439 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/C in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:33,439 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/cf14aa850e084555bce2c822a037561d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/7a3a081eeb574ca39e04b7486c6182e6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/eae20893234c4468a9b3f8cc98252f8f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/c9ba20a8e8a84cb9807c4668b6c7d9df] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=48.0 K 2024-11-18T20:41:33,441 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting cf14aa850e084555bce2c822a037561d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731962487731 2024-11-18T20:41:33,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742019_1195 (size=4469) 2024-11-18T20:41:33,441 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a3a081eeb574ca39e04b7486c6182e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731962487846 2024-11-18T20:41:33,442 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting eae20893234c4468a9b3f8cc98252f8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731962488490 2024-11-18T20:41:33,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742018_1194 (size=12454) 2024-11-18T20:41:33,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:33,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962553434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:33,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:33,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962553434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:33,443 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting c9ba20a8e8a84cb9807c4668b6c7d9df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731962490740 2024-11-18T20:41:33,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:33,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962553437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:33,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:33,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:33,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:33,457 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a35f0bb7923143469a95f183ac56a0aa_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a35f0bb7923143469a95f183ac56a0aa_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:33,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/d201ac14855540e4992b9c488192149d, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:33,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/d201ac14855540e4992b9c488192149d is 175, key is test_row_0/A:col10/1731962493296/Put/seqid=0 2024-11-18T20:41:33,464 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#C#compaction#164 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:33,465 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/37bd3b8420bf45ffb5035a566db9367e is 50, key is test_row_0/C:col10/1731962490740/Put/seqid=0 2024-11-18T20:41:33,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742020_1196 (size=31255) 2024-11-18T20:41:33,468 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=564, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/d201ac14855540e4992b9c488192149d 2024-11-18T20:41:33,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/217dc02ab83442ceb1bdf9ceda068fe2 is 50, key is test_row_0/B:col10/1731962493296/Put/seqid=0 2024-11-18T20:41:33,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742022_1198 (size=12301) 2024-11-18T20:41:33,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742021_1197 (size=12881) 2024-11-18T20:41:33,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962553551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:33,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962553551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:33,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962553551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:33,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:33,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962553767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:33,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:33,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962553767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:33,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962553768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:33,843 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#A#compaction#162 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:33,843 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/cf00735208c64a95a301aea93681e7de is 175, key is test_row_0/A:col10/1731962490740/Put/seqid=0 2024-11-18T20:41:33,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742023_1199 (size=31801) 2024-11-18T20:41:33,935 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/217dc02ab83442ceb1bdf9ceda068fe2 2024-11-18T20:41:33,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/f42019f27be4412f8ffce37de51855c5 is 50, key is test_row_0/C:col10/1731962493296/Put/seqid=0 2024-11-18T20:41:33,960 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/37bd3b8420bf45ffb5035a566db9367e as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/37bd3b8420bf45ffb5035a566db9367e 2024-11-18T20:41:33,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742024_1200 (size=12301) 2024-11-18T20:41:33,961 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/f42019f27be4412f8ffce37de51855c5 2024-11-18T20:41:33,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/d201ac14855540e4992b9c488192149d as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/d201ac14855540e4992b9c488192149d 2024-11-18T20:41:33,966 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/C of 59378c222c2195896b2cd9530f63ae92 into 37bd3b8420bf45ffb5035a566db9367e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:33,966 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:33,966 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/C, priority=12, startTime=1731962493349; duration=0sec 2024-11-18T20:41:33,967 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:33,967 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:C 2024-11-18T20:41:33,972 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/d201ac14855540e4992b9c488192149d, entries=150, sequenceid=564, filesize=30.5 K 2024-11-18T20:41:33,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/217dc02ab83442ceb1bdf9ceda068fe2 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/217dc02ab83442ceb1bdf9ceda068fe2 2024-11-18T20:41:33,977 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/217dc02ab83442ceb1bdf9ceda068fe2, entries=150, sequenceid=564, filesize=12.0 K 2024-11-18T20:41:33,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/f42019f27be4412f8ffce37de51855c5 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/f42019f27be4412f8ffce37de51855c5 2024-11-18T20:41:33,983 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/f42019f27be4412f8ffce37de51855c5, entries=150, sequenceid=564, filesize=12.0 K 2024-11-18T20:41:33,984 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~457.83 KB/468816, currentSize=20.13 KB/20610 for 59378c222c2195896b2cd9530f63ae92 in 623ms, sequenceid=564, compaction requested=false 2024-11-18T20:41:33,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:33,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:33,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-18T20:41:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-18T20:41:33,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-18T20:41:33,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7010 sec 2024-11-18T20:41:33,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.7080 sec 2024-11-18T20:41:34,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:34,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T20:41:34,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:34,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:34,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:34,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:34,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:34,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:34,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111823546677c9e4445385b7fbb2ebab2996_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962494081/Put/seqid=0 2024-11-18T20:41:34,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742025_1201 (size=17534) 2024-11-18T20:41:34,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:34,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962554121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:34,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731962494109, max timestamp=1731962494142Num uniques 859; 2024-11-18T20:41:34,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731962494109, max timestamp=1731962494142Num uniques 859; 2024-11-18T20:41:34,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-18T20:41:34,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-18T20:41:34,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:34,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731962494109, max timestamp=1731962494142Num uniques 859; 2024-11-18T20:41:34,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-18T20:41:34,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962554202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:34,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962554203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:34,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:34,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962554247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:34,259 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/cf00735208c64a95a301aea93681e7de as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/cf00735208c64a95a301aea93681e7de 2024-11-18T20:41:34,266 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/A of 59378c222c2195896b2cd9530f63ae92 into cf00735208c64a95a301aea93681e7de(size=31.1 K), total size for store is 61.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:34,266 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:34,266 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/A, priority=10, startTime=1731962493347; duration=0sec 2024-11-18T20:41:34,266 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:34,266 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:A 2024-11-18T20:41:34,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:34,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962554308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:34,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962554308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:34,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T20:41:34,437 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:34,438 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:41:34,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-18T20:41:34,440 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:41:34,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T20:41:34,441 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:41:34,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:41:34,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:34,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962554467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:34,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:34,502 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:34,507 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111823546677c9e4445385b7fbb2ebab2996_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111823546677c9e4445385b7fbb2ebab2996_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:34,508 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/7ee3f3361872420e8f3749d884c733ab, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:34,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/7ee3f3361872420e8f3749d884c733ab is 175, key is test_row_0/A:col10/1731962494081/Put/seqid=0 2024-11-18T20:41:34,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742026_1202 (size=48639) 2024-11-18T20:41:34,514 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=577, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/7ee3f3361872420e8f3749d884c733ab 2024-11-18T20:41:34,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962554517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:34,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962554517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:34,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/f4d8be1a8e72473bad4502526889955f is 50, key is test_row_0/B:col10/1731962494081/Put/seqid=0 2024-11-18T20:41:34,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742027_1203 (size=12301) 2024-11-18T20:41:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T20:41:34,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-18T20:41:34,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:34,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:34,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:34,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:34,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:34,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-18T20:41:34,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:34,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:34,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:34,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:34,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T20:41:34,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962554777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:34,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962554827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:34,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:34,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:34,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962554827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:34,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:34,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:34,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-18T20:41:34,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:34,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:34,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:34,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:34,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:34,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/f4d8be1a8e72473bad4502526889955f 2024-11-18T20:41:34,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/193a4790063348b69f71d7aea1951bee is 50, key is test_row_0/C:col10/1731962494081/Put/seqid=0 2024-11-18T20:41:34,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742028_1204 (size=12301) 2024-11-18T20:41:35,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-18T20:41:35,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:35,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:35,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:35,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:35,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T20:41:35,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-18T20:41:35,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:35,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:35,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:35,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:35,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:35,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962555297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:35,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:35,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:35,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:35,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/193a4790063348b69f71d7aea1951bee 2024-11-18T20:41:35,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962555337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:35,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962555337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:35,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:35,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:35,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/7ee3f3361872420e8f3749d884c733ab as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/7ee3f3361872420e8f3749d884c733ab 2024-11-18T20:41:35,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/7ee3f3361872420e8f3749d884c733ab, entries=250, sequenceid=577, filesize=47.5 K 2024-11-18T20:41:35,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-18T20:41:35,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:35,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:35,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:35,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:35,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:35,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/f4d8be1a8e72473bad4502526889955f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/f4d8be1a8e72473bad4502526889955f 2024-11-18T20:41:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:35,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/f4d8be1a8e72473bad4502526889955f, entries=150, sequenceid=577, filesize=12.0 K 2024-11-18T20:41:35,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/193a4790063348b69f71d7aea1951bee as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/193a4790063348b69f71d7aea1951bee 2024-11-18T20:41:35,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/193a4790063348b69f71d7aea1951bee, entries=150, sequenceid=577, filesize=12.0 K 2024-11-18T20:41:35,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for 59378c222c2195896b2cd9530f63ae92 in 1297ms, sequenceid=577, compaction requested=true 2024-11-18T20:41:35,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:35,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:41:35,381 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:35,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:35,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:41:35,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:35,381 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:35,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:41:35,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:35,383 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111695 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:35,383 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:35,383 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/B is initiating minor compaction (all files) 2024-11-18T20:41:35,383 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/A is initiating minor compaction (all files) 2024-11-18T20:41:35,383 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/B in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:35,383 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/A in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:35,383 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/cf00735208c64a95a301aea93681e7de, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/d201ac14855540e4992b9c488192149d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/7ee3f3361872420e8f3749d884c733ab] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=109.1 K 2024-11-18T20:41:35,383 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/2d8d8dfe2810420d83fe9e38cd2b1313, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/217dc02ab83442ceb1bdf9ceda068fe2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/f4d8be1a8e72473bad4502526889955f] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=36.6 K 2024-11-18T20:41:35,383 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:35,383 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/cf00735208c64a95a301aea93681e7de, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/d201ac14855540e4992b9c488192149d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/7ee3f3361872420e8f3749d884c733ab] 2024-11-18T20:41:35,383 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d8d8dfe2810420d83fe9e38cd2b1313, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731962490740 2024-11-18T20:41:35,384 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf00735208c64a95a301aea93681e7de, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731962490740 2024-11-18T20:41:35,384 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 217dc02ab83442ceb1bdf9ceda068fe2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731962493296 2024-11-18T20:41:35,385 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting d201ac14855540e4992b9c488192149d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731962493296 2024-11-18T20:41:35,385 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting f4d8be1a8e72473bad4502526889955f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1731962494079 2024-11-18T20:41:35,385 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ee3f3361872420e8f3749d884c733ab, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1731962494078 2024-11-18T20:41:35,394 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:35,395 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#B#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:35,396 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/3692231269f1489dbd5ae45ee027d15b is 50, key is test_row_0/B:col10/1731962494081/Put/seqid=0 2024-11-18T20:41:35,401 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118a1ebb3c4beed43fdbff994841ecd8866_59378c222c2195896b2cd9530f63ae92 store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:35,404 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118a1ebb3c4beed43fdbff994841ecd8866_59378c222c2195896b2cd9530f63ae92, store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:35,404 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a1ebb3c4beed43fdbff994841ecd8866_59378c222c2195896b2cd9530f63ae92 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:35,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742029_1205 (size=12983) 2024-11-18T20:41:35,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742030_1206 (size=4469) 2024-11-18T20:41:35,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-18T20:41:35,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:35,520 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-18T20:41:35,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:35,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:35,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:35,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:35,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:35,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:35,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118357501ed18e14d8a9741dd6820a3cf5b_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962494187/Put/seqid=0 2024-11-18T20:41:35,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742031_1207 (size=12454) 2024-11-18T20:41:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T20:41:35,820 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/3692231269f1489dbd5ae45ee027d15b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3692231269f1489dbd5ae45ee027d15b 2024-11-18T20:41:35,820 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#A#compaction#171 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:35,820 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/cbb01ef29708433b9fcb1c2ca05a3a9b is 175, key is test_row_0/A:col10/1731962494081/Put/seqid=0 2024-11-18T20:41:35,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742032_1208 (size=31903) 2024-11-18T20:41:35,828 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/B of 59378c222c2195896b2cd9530f63ae92 into 3692231269f1489dbd5ae45ee027d15b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:35,828 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:35,829 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/B, priority=13, startTime=1731962495381; duration=0sec 2024-11-18T20:41:35,829 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:35,829 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:B 2024-11-18T20:41:35,829 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:35,830 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:35,830 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/C is initiating minor compaction (all files) 2024-11-18T20:41:35,830 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/C in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:35,830 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/37bd3b8420bf45ffb5035a566db9367e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/f42019f27be4412f8ffce37de51855c5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/193a4790063348b69f71d7aea1951bee] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=36.6 K 2024-11-18T20:41:35,834 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 37bd3b8420bf45ffb5035a566db9367e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731962490740 2024-11-18T20:41:35,834 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting f42019f27be4412f8ffce37de51855c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731962493296 2024-11-18T20:41:35,835 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 193a4790063348b69f71d7aea1951bee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1731962494079 2024-11-18T20:41:35,841 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#C#compaction#173 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:35,841 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/03a81984b0f04811b4b8ae1902f8029f is 50, key is test_row_0/C:col10/1731962494081/Put/seqid=0 2024-11-18T20:41:35,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742033_1209 (size=12983) 2024-11-18T20:41:35,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:35,939 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118357501ed18e14d8a9741dd6820a3cf5b_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118357501ed18e14d8a9741dd6820a3cf5b_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:35,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/107bfe04ae954b27925f3cc615272607, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:35,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/107bfe04ae954b27925f3cc615272607 is 175, key is test_row_0/A:col10/1731962494187/Put/seqid=0 2024-11-18T20:41:35,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742034_1210 (size=31255) 2024-11-18T20:41:35,945 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=612, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/107bfe04ae954b27925f3cc615272607 2024-11-18T20:41:35,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/3d35953bb43443d0bf9138823c71b83e is 50, key is test_row_0/B:col10/1731962494187/Put/seqid=0 2024-11-18T20:41:35,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742035_1211 (size=12301) 2024-11-18T20:41:36,233 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/cbb01ef29708433b9fcb1c2ca05a3a9b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/cbb01ef29708433b9fcb1c2ca05a3a9b 2024-11-18T20:41:36,239 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/A of 59378c222c2195896b2cd9530f63ae92 into cbb01ef29708433b9fcb1c2ca05a3a9b(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:36,239 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:36,239 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/A, priority=13, startTime=1731962495381; duration=0sec 2024-11-18T20:41:36,239 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:36,239 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:A 2024-11-18T20:41:36,250 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/03a81984b0f04811b4b8ae1902f8029f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/03a81984b0f04811b4b8ae1902f8029f 2024-11-18T20:41:36,267 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/C of 59378c222c2195896b2cd9530f63ae92 into 03a81984b0f04811b4b8ae1902f8029f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:36,267 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:36,267 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/C, priority=13, startTime=1731962495381; duration=0sec 2024-11-18T20:41:36,267 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:36,267 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:C 2024-11-18T20:41:36,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:36,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962556342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:36,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962556357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:36,359 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 59378c222c2195896b2cd9530f63ae92 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:36,362 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=612 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/3d35953bb43443d0bf9138823c71b83e 2024-11-18T20:41:36,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962556367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:36,372 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in 59378c222c2195896b2cd9530f63ae92 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:36,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:36,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:36,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:36,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/3d6b1bd21145471dbf85bda9ee61efd4 is 50, key is test_row_0/C:col10/1731962494187/Put/seqid=0 2024-11-18T20:41:36,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742036_1212 (size=12301) 2024-11-18T20:41:36,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962556457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:36,458 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:36,458 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:36,458 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T20:41:36,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962556668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:36,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:36,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:36,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:36,782 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=612 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/3d6b1bd21145471dbf85bda9ee61efd4 2024-11-18T20:41:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/107bfe04ae954b27925f3cc615272607 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/107bfe04ae954b27925f3cc615272607 2024-11-18T20:41:36,792 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/107bfe04ae954b27925f3cc615272607, entries=150, sequenceid=612, filesize=30.5 K 2024-11-18T20:41:36,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/3d35953bb43443d0bf9138823c71b83e as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3d35953bb43443d0bf9138823c71b83e 2024-11-18T20:41:36,798 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3d35953bb43443d0bf9138823c71b83e, entries=150, sequenceid=612, filesize=12.0 K 2024-11-18T20:41:36,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/3d6b1bd21145471dbf85bda9ee61efd4 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/3d6b1bd21145471dbf85bda9ee61efd4 2024-11-18T20:41:36,805 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/3d6b1bd21145471dbf85bda9ee61efd4, entries=150, sequenceid=612, filesize=12.0 K 2024-11-18T20:41:36,807 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for 59378c222c2195896b2cd9530f63ae92 in 1287ms, sequenceid=612, compaction requested=false 2024-11-18T20:41:36,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:36,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:36,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-18T20:41:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-18T20:41:36,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-18T20:41:36,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3680 sec 2024-11-18T20:41:36,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.3730 sec 2024-11-18T20:41:36,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:36,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T20:41:36,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:36,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:36,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:36,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:36,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:36,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:36,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118031fb51c265e403dbf26f2f4ad9e410f_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962496335/Put/seqid=0 2024-11-18T20:41:36,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742037_1213 (size=14994) 2024-11-18T20:41:37,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962497050, max timestamp=1731962497066Num uniques 859; 2024-11-18T20:41:37,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:41:37,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962497050, max timestamp=1731962497066Num uniques 859; 2024-11-18T20:41:37,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962497050, max timestamp=1731962497066Num uniques 859; 2024-11-18T20:41:37,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:41:37,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:41:37,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962557171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:37,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:37,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:37,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:37,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962557287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:37,392 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:37,399 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118031fb51c265e403dbf26f2f4ad9e410f_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118031fb51c265e403dbf26f2f4ad9e410f_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:37,400 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/767fb9d23c224f30a4b4d42b0b224111, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:37,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/767fb9d23c224f30a4b4d42b0b224111 is 175, key is test_row_0/A:col10/1731962496335/Put/seqid=0 2024-11-18T20:41:37,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742038_1214 (size=39949) 2024-11-18T20:41:37,410 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=626, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/767fb9d23c224f30a4b4d42b0b224111 2024-11-18T20:41:37,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/c2937c47d1234eaab0b9c1187fbed3bf is 50, key is test_row_0/B:col10/1731962496335/Put/seqid=0 2024-11-18T20:41:37,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742039_1215 (size=12301) 2024-11-18T20:41:37,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962557497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:37,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962557807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:37,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/c2937c47d1234eaab0b9c1187fbed3bf 2024-11-18T20:41:37,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/61c279914aad49cba0bf46a4ba0395fc is 50, key is test_row_0/C:col10/1731962496335/Put/seqid=0 2024-11-18T20:41:37,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742040_1216 (size=12301) 2024-11-18T20:41:38,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/61c279914aad49cba0bf46a4ba0395fc 2024-11-18T20:41:38,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/767fb9d23c224f30a4b4d42b0b224111 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/767fb9d23c224f30a4b4d42b0b224111 2024-11-18T20:41:38,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/767fb9d23c224f30a4b4d42b0b224111, entries=200, sequenceid=626, filesize=39.0 K 2024-11-18T20:41:38,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/c2937c47d1234eaab0b9c1187fbed3bf as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/c2937c47d1234eaab0b9c1187fbed3bf 2024-11-18T20:41:38,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/c2937c47d1234eaab0b9c1187fbed3bf, entries=150, sequenceid=626, filesize=12.0 K 2024-11-18T20:41:38,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/61c279914aad49cba0bf46a4ba0395fc as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/61c279914aad49cba0bf46a4ba0395fc 2024-11-18T20:41:38,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:38,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/61c279914aad49cba0bf46a4ba0395fc, entries=150, sequenceid=626, filesize=12.0 K 2024-11-18T20:41:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962558318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:38,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 59378c222c2195896b2cd9530f63ae92 in 1351ms, sequenceid=626, compaction requested=true 2024-11-18T20:41:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:38,332 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:41:38,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:38,333 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:38,333 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:38,333 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/A is initiating minor compaction (all files) 2024-11-18T20:41:38,333 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/A in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:38,333 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/cbb01ef29708433b9fcb1c2ca05a3a9b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/107bfe04ae954b27925f3cc615272607, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/767fb9d23c224f30a4b4d42b0b224111] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=100.7 K 2024-11-18T20:41:38,333 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:38,334 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/cbb01ef29708433b9fcb1c2ca05a3a9b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/107bfe04ae954b27925f3cc615272607, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/767fb9d23c224f30a4b4d42b0b224111] 2024-11-18T20:41:38,334 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:38,334 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/B is initiating minor compaction (all files) 2024-11-18T20:41:38,334 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/B in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:38,334 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3692231269f1489dbd5ae45ee027d15b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3d35953bb43443d0bf9138823c71b83e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/c2937c47d1234eaab0b9c1187fbed3bf] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=36.7 K 2024-11-18T20:41:38,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:41:38,335 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbb01ef29708433b9fcb1c2ca05a3a9b, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1731962494079 2024-11-18T20:41:38,335 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 3692231269f1489dbd5ae45ee027d15b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1731962494079 2024-11-18T20:41:38,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:38,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:41:38,335 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 107bfe04ae954b27925f3cc615272607, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=612, earliestPutTs=1731962494173 2024-11-18T20:41:38,336 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d35953bb43443d0bf9138823c71b83e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=612, earliestPutTs=1731962494173 2024-11-18T20:41:38,336 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting c2937c47d1234eaab0b9c1187fbed3bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1731962496335 2024-11-18T20:41:38,336 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 767fb9d23c224f30a4b4d42b0b224111, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1731962496324 2024-11-18T20:41:38,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:38,356 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#B#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:38,358 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/206abc8da02245c3b28d2e15dcdc1d5b is 50, key is test_row_0/B:col10/1731962496335/Put/seqid=0 2024-11-18T20:41:38,365 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:38,386 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118f58f21ec63044a00a512537a3bfeb7a9_59378c222c2195896b2cd9530f63ae92 store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:38,388 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118f58f21ec63044a00a512537a3bfeb7a9_59378c222c2195896b2cd9530f63ae92, store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:38,388 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f58f21ec63044a00a512537a3bfeb7a9_59378c222c2195896b2cd9530f63ae92 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:38,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-18T20:41:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:38,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731962497112, max timestamp=1731962498395Num uniques 1018; 2024-11-18T20:41:38,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-18T20:41:38,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731962497112, max timestamp=1731962498395Num uniques 1018; 2024-11-18T20:41:38,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-18T20:41:38,408 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:41:38,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731962497112, max timestamp=1731962498395Num uniques 1018; 2024-11-18T20:41:38,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742041_1217 (size=13085) 2024-11-18T20:41:38,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-18T20:41:38,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:38,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:38,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:38,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:38,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:38,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:38,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742042_1218 (size=4469) 2024-11-18T20:41:38,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111876e8bc666e3a4120b8ccf659ca5767a4_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962498395/Put/seqid=0 2024-11-18T20:41:38,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742043_1219 (size=17534) 2024-11-18T20:41:38,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962498498, max timestamp=1731962498566Num uniques 859; 2024-11-18T20:41:38,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:41:38,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962498498, max timestamp=1731962498566Num uniques 859; 2024-11-18T20:41:38,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:41:38,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962498523, max timestamp=1731962498566Num uniques 859; 2024-11-18T20:41:38,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:41:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T20:41:38,608 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:38,610 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:41:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-18T20:41:38,619 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:41:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T20:41:38,620 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:41:38,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:41:38,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731962498627, max timestamp=1731962498672Num uniques 1068; 2024-11-18T20:41:38,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-18T20:41:38,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731962498627, max timestamp=1731962498672Num uniques 1068; 2024-11-18T20:41:38,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-18T20:41:38,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731962498627, max timestamp=1731962498672Num uniques 1018; 2024-11-18T20:41:38,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-18T20:41:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T20:41:38,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-18T20:41:38,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:38,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:38,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:38,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:38,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:38,826 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/206abc8da02245c3b28d2e15dcdc1d5b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/206abc8da02245c3b28d2e15dcdc1d5b 2024-11-18T20:41:38,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731962498760, max timestamp=1731962498788Num uniques 1077; 2024-11-18T20:41:38,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-18T20:41:38,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731962498760, max timestamp=1731962498788Num uniques 1077; 2024-11-18T20:41:38,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-18T20:41:38,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731962498760, max timestamp=1731962498788Num uniques 1077; 2024-11-18T20:41:38,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-18T20:41:38,843 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/B of 59378c222c2195896b2cd9530f63ae92 into 206abc8da02245c3b28d2e15dcdc1d5b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:38,843 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:38,843 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/B, priority=13, startTime=1731962498333; duration=0sec 2024-11-18T20:41:38,843 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:38,843 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:B 2024-11-18T20:41:38,844 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:38,844 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#A#compaction#180 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:38,845 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/2bdb2404495b4bffa45737a37b70a8f0 is 175, key is test_row_0/A:col10/1731962496335/Put/seqid=0 2024-11-18T20:41:38,850 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:38,850 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/C is initiating minor compaction (all files) 2024-11-18T20:41:38,850 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/C in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:38,850 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/03a81984b0f04811b4b8ae1902f8029f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/3d6b1bd21145471dbf85bda9ee61efd4, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/61c279914aad49cba0bf46a4ba0395fc] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=36.7 K 2024-11-18T20:41:38,851 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 03a81984b0f04811b4b8ae1902f8029f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1731962494079 2024-11-18T20:41:38,851 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d6b1bd21145471dbf85bda9ee61efd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=612, earliestPutTs=1731962494173 2024-11-18T20:41:38,852 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 61c279914aad49cba0bf46a4ba0395fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1731962496335 2024-11-18T20:41:38,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742044_1220 (size=32005) 2024-11-18T20:41:38,869 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#C#compaction#182 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:38,869 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/2bdb2404495b4bffa45737a37b70a8f0 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/2bdb2404495b4bffa45737a37b70a8f0 2024-11-18T20:41:38,870 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/6d13b261fa2e4fa58fd5b457bc82ccda is 50, key is test_row_0/C:col10/1731962496335/Put/seqid=0 2024-11-18T20:41:38,880 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/A of 59378c222c2195896b2cd9530f63ae92 into 2bdb2404495b4bffa45737a37b70a8f0(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:38,880 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:38,880 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/A, priority=13, startTime=1731962498332; duration=0sec 2024-11-18T20:41:38,881 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:38,881 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:A 2024-11-18T20:41:38,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742045_1221 (size=13085) 2024-11-18T20:41:38,915 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:38,922 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111876e8bc666e3a4120b8ccf659ca5767a4_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111876e8bc666e3a4120b8ccf659ca5767a4_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:38,923 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/6d13b261fa2e4fa58fd5b457bc82ccda as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/6d13b261fa2e4fa58fd5b457bc82ccda 2024-11-18T20:41:38,924 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/5b256d7d96654451a1fbcb7cbe0f524f, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:38,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/5b256d7d96654451a1fbcb7cbe0f524f is 175, key is test_row_0/A:col10/1731962498395/Put/seqid=0 2024-11-18T20:41:38,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-18T20:41:38,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:38,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:38,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:38,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:38,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:38,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:38,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T20:41:38,941 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/C of 59378c222c2195896b2cd9530f63ae92 into 6d13b261fa2e4fa58fd5b457bc82ccda(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:38,941 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:38,942 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/C, priority=13, startTime=1731962498335; duration=0sec 2024-11-18T20:41:38,942 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:38,942 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:C 2024-11-18T20:41:38,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731962498892, max timestamp=1731962498941Num uniques 1086; 2024-11-18T20:41:38,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-18T20:41:38,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731962498892, max timestamp=1731962498941Num uniques 1086; 2024-11-18T20:41:38,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-18T20:41:38,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731962498892, max timestamp=1731962498941Num uniques 1086; 2024-11-18T20:41:38,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-18T20:41:38,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742046_1222 (size=48639) 2024-11-18T20:41:39,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731962499010, max timestamp=1731962499031Num uniques 1195; 2024-11-18T20:41:39,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731962499010, max timestamp=1731962499031Num uniques 1195; 2024-11-18T20:41:39,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=8930, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-11-18T20:41:39,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=8930, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-11-18T20:41:39,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8924, totalHeapSize=20520, min timestamp=1731962499010, max timestamp=1731962499032Num uniques 1195; 2024-11-18T20:41:39,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54724, new segment data size=8924, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-11-18T20:41:39,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-18T20:41:39,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:39,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:39,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:39,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:39,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:39,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731962499073, max timestamp=1731962499120Num uniques 1054; 2024-11-18T20:41:39,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-18T20:41:39,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731962499073, max timestamp=1731962499120Num uniques 1054; 2024-11-18T20:41:39,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-18T20:41:39,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731962499073, max timestamp=1731962499120Num uniques 1054; 2024-11-18T20:41:39,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-18T20:41:39,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731962499160, max timestamp=1731962499173Num uniques 1113; 2024-11-18T20:41:39,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731962499160, max timestamp=1731962499173Num uniques 1113; 2024-11-18T20:41:39,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-11-18T20:41:39,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-11-18T20:41:39,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731962499160, max timestamp=1731962499173Num uniques 1013; 2024-11-18T20:41:39,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-18T20:41:39,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731962499182, max timestamp=1731962499224Num uniques 1022; 2024-11-18T20:41:39,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-18T20:41:39,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731962499182, max timestamp=1731962499224Num uniques 1022; 2024-11-18T20:41:39,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-18T20:41:39,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731962499182, max timestamp=1731962499224Num uniques 1022; 2024-11-18T20:41:39,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-18T20:41:39,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-18T20:41:39,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:39,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:39,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:39,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:39,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T20:41:39,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8276, totalHeapSize=19064, min timestamp=1731962499280, max timestamp=1731962499300Num uniques 1081; 2024-11-18T20:41:39,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49496, new segment data size=8276, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-18T20:41:39,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8276, totalHeapSize=19064, min timestamp=1731962499280, max timestamp=1731962499300Num uniques 1081; 2024-11-18T20:41:39,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49496, new segment data size=8276, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-18T20:41:39,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8282, totalHeapSize=19064, min timestamp=1731962499280, max timestamp=1731962499300Num uniques 1081; 2024-11-18T20:41:39,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49502, new segment data size=8282, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-18T20:41:39,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731962499313, max timestamp=1731962499350Num uniques 1040; 2024-11-18T20:41:39,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-18T20:41:39,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731962499313, max timestamp=1731962499350Num uniques 1040; 2024-11-18T20:41:39,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-18T20:41:39,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731962499344, max timestamp=1731962499350Num uniques 1040; 2024-11-18T20:41:39,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=6870, suffix heap size=136128,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=150 2024-11-18T20:41:39,358 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=664, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/5b256d7d96654451a1fbcb7cbe0f524f 2024-11-18T20:41:39,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/56bca814c25e4bf596627301e517c5c0 is 50, key is test_row_0/B:col10/1731962498395/Put/seqid=0 2024-11-18T20:41:39,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742047_1223 (size=12301) 2024-11-18T20:41:39,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/56bca814c25e4bf596627301e517c5c0 2024-11-18T20:41:39,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-18T20:41:39,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:39,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:39,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:39,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:39,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:39,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731962499381, max timestamp=1731962499391Num uniques 1099; 2024-11-18T20:41:39,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=13694, suffix heap size=142264,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=299 2024-11-18T20:41:39,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731962499381, max timestamp=1731962499391Num uniques 1009; 2024-11-18T20:41:39,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=13694, suffix heap size=132904,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=299 2024-11-18T20:41:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:39,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/4bd21bd512254cb79eac24ff07a2b940 is 50, key is test_row_0/C:col10/1731962498395/Put/seqid=0 2024-11-18T20:41:39,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731962499381, max timestamp=1731962499391Num uniques 1099; 2024-11-18T20:41:39,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=13694, suffix heap size=142264,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=299 2024-11-18T20:41:39,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742048_1224 (size=12301) 2024-11-18T20:41:39,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962559474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:39,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:39,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 340 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962559474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:39,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:39,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:39,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:39,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:39,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:39,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962559477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:39,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:39,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:39,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:39,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-18T20:41:39,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:39,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:39,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:39,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:39,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:39,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:39,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962559588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 315 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962559588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:39,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 341 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962559588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:39,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-18T20:41:39,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:39,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:39,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:39,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:39,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T20:41:39,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 316 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962559797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:39,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 342 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962559797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:39,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962559797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:39,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:39,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:39,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:39,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/4bd21bd512254cb79eac24ff07a2b940 2024-11-18T20:41:39,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/5b256d7d96654451a1fbcb7cbe0f524f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/5b256d7d96654451a1fbcb7cbe0f524f 2024-11-18T20:41:39,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/5b256d7d96654451a1fbcb7cbe0f524f, entries=250, sequenceid=664, filesize=47.5 K 2024-11-18T20:41:39,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/56bca814c25e4bf596627301e517c5c0 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/56bca814c25e4bf596627301e517c5c0 2024-11-18T20:41:39,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/56bca814c25e4bf596627301e517c5c0, entries=150, sequenceid=664, filesize=12.0 K 2024-11-18T20:41:39,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/4bd21bd512254cb79eac24ff07a2b940 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/4bd21bd512254cb79eac24ff07a2b940 2024-11-18T20:41:39,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-18T20:41:39,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:39,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/4bd21bd512254cb79eac24ff07a2b940, entries=150, sequenceid=664, filesize=12.0 K 2024-11-18T20:41:39,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:39,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:39,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:39,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:39,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=154.31 KB/158010 for 59378c222c2195896b2cd9530f63ae92 in 1481ms, sequenceid=664, compaction requested=false 2024-11-18T20:41:39,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:40,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-18T20:41:40,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:40,040 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=154.31 KB heapSize=431.63 KB 2024-11-18T20:41:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:40,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111891b84e28abef4aee87bb5c20da5934b8_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962499463/Put/seqid=0 2024-11-18T20:41:40,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742049_1225 (size=12454) 2024-11-18T20:41:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:40,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. as already flushing 2024-11-18T20:41:40,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962560112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:40,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 344 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962560112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:40,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 319 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962560114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:40,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:40,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962560218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:40,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 345 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962560218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:40,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962560227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:40,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:40,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:40,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962560429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:40,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:40,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:40,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 346 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962560429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:40,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:40,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:40,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 321 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962560439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:40,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:40,487 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-18T20:41:40,487 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:41:40,487 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:40,488 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:40,488 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111891b84e28abef4aee87bb5c20da5934b8_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111891b84e28abef4aee87bb5c20da5934b8_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:40,489 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-11-18T20:41:40,489 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:41:40,489 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:40,488 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:40,489 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:40,489 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:40,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/9cced5c9702a4a069865b2b0c53f4a1e, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:40,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/9cced5c9702a4a069865b2b0c53f4a1e is 175, key is test_row_0/A:col10/1731962499463/Put/seqid=0 2024-11-18T20:41:40,493 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-18T20:41:40,493 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:41:40,493 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:40,493 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:40,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:40,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742050_1226 (size=31255) 2024-11-18T20:41:40,504 INFO [Thread-652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-652. 2024-11-18T20:41:40,504 DEBUG [Thread-652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:41:40,505 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:40,505 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:40,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:40,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:40,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962560738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:40,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:40,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 322 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962560747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 347 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962560747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:40,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T20:41:40,899 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=876, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/9cced5c9702a4a069865b2b0c53f4a1e 2024-11-18T20:41:40,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/84afaa1e732343fa9e4aad617d715356 is 50, key is test_row_0/B:col10/1731962499463/Put/seqid=0 2024-11-18T20:41:40,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742051_1227 (size=12301) 2024-11-18T20:41:41,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731962561038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:41,038 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in 59378c222c2195896b2cd9530f63ae92 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18275 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:41,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:41,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:41,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:41,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43748 deadline: 1731962561077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:41,079 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 59378c222c2195896b2cd9530f63ae92 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18315 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:41,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:41,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43682 deadline: 1731962561247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:41,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:41,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:41,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 348 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43696 deadline: 1731962561257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:41,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43720 deadline: 1731962561257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:41:41,315 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=876 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/84afaa1e732343fa9e4aad617d715356 2024-11-18T20:41:41,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/a1568e7022304a13b3d1d25d720ea9cb is 50, key is test_row_0/C:col10/1731962499463/Put/seqid=0 2024-11-18T20:41:41,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742052_1228 (size=12301) 2024-11-18T20:41:41,727 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=876 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/a1568e7022304a13b3d1d25d720ea9cb 2024-11-18T20:41:41,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/9cced5c9702a4a069865b2b0c53f4a1e as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/9cced5c9702a4a069865b2b0c53f4a1e 2024-11-18T20:41:41,739 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/9cced5c9702a4a069865b2b0c53f4a1e, entries=150, sequenceid=876, filesize=30.5 K 2024-11-18T20:41:41,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/84afaa1e732343fa9e4aad617d715356 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/84afaa1e732343fa9e4aad617d715356 2024-11-18T20:41:41,748 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/84afaa1e732343fa9e4aad617d715356, entries=150, sequenceid=876, filesize=12.0 K 2024-11-18T20:41:41,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/a1568e7022304a13b3d1d25d720ea9cb as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/a1568e7022304a13b3d1d25d720ea9cb 2024-11-18T20:41:41,753 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/a1568e7022304a13b3d1d25d720ea9cb, entries=150, sequenceid=876, filesize=12.0 K 2024-11-18T20:41:41,755 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~431.58 KB/441936, currentSize=26.84 KB/27480 for 59378c222c2195896b2cd9530f63ae92 in 1714ms, sequenceid=876, compaction requested=true 2024-11-18T20:41:41,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:41,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:41,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-18T20:41:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-18T20:41:41,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-18T20:41:41,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1360 sec 2024-11-18T20:41:41,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 3.1490 sec 2024-11-18T20:41:42,260 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-11-18T20:41:42,260 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:41:42,260 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:42,260 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:42,260 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:42,270 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-18T20:41:42,270 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:41:42,270 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:42,270 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:42,270 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:42,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:42,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T20:41:42,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:42,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:42,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:42,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:42,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:42,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:42,279 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-18T20:41:42,279 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:41:42,279 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:42,279 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:42,279 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:42,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d512d0f71105491088cdccf0c2dd5e1f_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962500112/Put/seqid=0 2024-11-18T20:41:42,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742053_1229 (size=12454) 2024-11-18T20:41:42,693 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:42,698 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d512d0f71105491088cdccf0c2dd5e1f_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d512d0f71105491088cdccf0c2dd5e1f_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:42,699 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/0ec9319c8adb4a6ebd7ba05574c31c7f, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:42,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/0ec9319c8adb4a6ebd7ba05574c31c7f is 175, key is test_row_0/A:col10/1731962500112/Put/seqid=0 2024-11-18T20:41:42,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742054_1230 (size=31255) 2024-11-18T20:41:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T20:41:42,788 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:43,104 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=886, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/0ec9319c8adb4a6ebd7ba05574c31c7f 2024-11-18T20:41:43,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/9c2f6b6505dd4c578602d93d8ea342bb is 50, key is test_row_0/B:col10/1731962500112/Put/seqid=0 2024-11-18T20:41:43,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742055_1231 (size=12301) 2024-11-18T20:41:43,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=886 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/9c2f6b6505dd4c578602d93d8ea342bb 2024-11-18T20:41:43,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/62fde375e11b4b01932836e6363c7080 is 50, key is test_row_0/C:col10/1731962500112/Put/seqid=0 2024-11-18T20:41:43,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742056_1232 (size=12301) 2024-11-18T20:41:43,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=886 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/62fde375e11b4b01932836e6363c7080 2024-11-18T20:41:43,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/0ec9319c8adb4a6ebd7ba05574c31c7f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/0ec9319c8adb4a6ebd7ba05574c31c7f 2024-11-18T20:41:43,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/0ec9319c8adb4a6ebd7ba05574c31c7f, entries=150, sequenceid=886, filesize=30.5 K 2024-11-18T20:41:43,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/9c2f6b6505dd4c578602d93d8ea342bb as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/9c2f6b6505dd4c578602d93d8ea342bb 2024-11-18T20:41:43,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/9c2f6b6505dd4c578602d93d8ea342bb, entries=150, sequenceid=886, filesize=12.0 K 2024-11-18T20:41:43,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/62fde375e11b4b01932836e6363c7080 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/62fde375e11b4b01932836e6363c7080 2024-11-18T20:41:43,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/62fde375e11b4b01932836e6363c7080, entries=150, sequenceid=886, filesize=12.0 K 2024-11-18T20:41:43,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 59378c222c2195896b2cd9530f63ae92 in 1285ms, sequenceid=886, compaction requested=true 2024-11-18T20:41:43,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:43,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:41:43,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:43,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:41:43,563 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:41:43,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:43,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59378c222c2195896b2cd9530f63ae92:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:41:43,563 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:41:43,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:43,564 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:41:43,564 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:41:43,564 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/B is initiating minor compaction (all files) 2024-11-18T20:41:43,564 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/A is initiating minor compaction (all files) 2024-11-18T20:41:43,564 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/B in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:43,564 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/A in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:43,564 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/206abc8da02245c3b28d2e15dcdc1d5b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/56bca814c25e4bf596627301e517c5c0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/84afaa1e732343fa9e4aad617d715356, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/9c2f6b6505dd4c578602d93d8ea342bb] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=48.8 K 2024-11-18T20:41:43,564 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/2bdb2404495b4bffa45737a37b70a8f0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/5b256d7d96654451a1fbcb7cbe0f524f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/9cced5c9702a4a069865b2b0c53f4a1e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/0ec9319c8adb4a6ebd7ba05574c31c7f] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=139.8 K 2024-11-18T20:41:43,564 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:43,564 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/2bdb2404495b4bffa45737a37b70a8f0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/5b256d7d96654451a1fbcb7cbe0f524f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/9cced5c9702a4a069865b2b0c53f4a1e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/0ec9319c8adb4a6ebd7ba05574c31c7f] 2024-11-18T20:41:43,565 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bdb2404495b4bffa45737a37b70a8f0, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1731962496335 2024-11-18T20:41:43,565 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 206abc8da02245c3b28d2e15dcdc1d5b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1731962496335 2024-11-18T20:41:43,565 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b256d7d96654451a1fbcb7cbe0f524f, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1731962497112 2024-11-18T20:41:43,565 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 56bca814c25e4bf596627301e517c5c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1731962497112 2024-11-18T20:41:43,565 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cced5c9702a4a069865b2b0c53f4a1e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=876, earliestPutTs=1731962499463 2024-11-18T20:41:43,566 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 84afaa1e732343fa9e4aad617d715356, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=876, earliestPutTs=1731962499463 2024-11-18T20:41:43,566 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ec9319c8adb4a6ebd7ba05574c31c7f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=886, earliestPutTs=1731962500112 2024-11-18T20:41:43,566 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c2f6b6505dd4c578602d93d8ea342bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=886, earliestPutTs=1731962500112 2024-11-18T20:41:43,578 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#B#compaction#191 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:43,578 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/2f234d63e7e64639a9ec159326699a77 is 50, key is test_row_0/B:col10/1731962500112/Put/seqid=0 2024-11-18T20:41:43,581 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:43,583 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111862d7a4e9540e41eaa6f4303f3801b4c7_59378c222c2195896b2cd9530f63ae92 store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:43,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742057_1233 (size=13221) 2024-11-18T20:41:43,621 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111862d7a4e9540e41eaa6f4303f3801b4c7_59378c222c2195896b2cd9530f63ae92, store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:43,622 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111862d7a4e9540e41eaa6f4303f3801b4c7_59378c222c2195896b2cd9530f63ae92 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:43,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742058_1234 (size=4469) 2024-11-18T20:41:43,992 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/2f234d63e7e64639a9ec159326699a77 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/2f234d63e7e64639a9ec159326699a77 2024-11-18T20:41:43,998 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/B of 59378c222c2195896b2cd9530f63ae92 into 2f234d63e7e64639a9ec159326699a77(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:43,998 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:43,998 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/B, priority=12, startTime=1731962503563; duration=0sec 2024-11-18T20:41:43,998 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:43,998 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:B 2024-11-18T20:41:43,998 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:41:44,000 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:41:44,000 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 59378c222c2195896b2cd9530f63ae92/C is initiating minor compaction (all files) 2024-11-18T20:41:44,000 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59378c222c2195896b2cd9530f63ae92/C in TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:44,000 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/6d13b261fa2e4fa58fd5b457bc82ccda, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/4bd21bd512254cb79eac24ff07a2b940, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/a1568e7022304a13b3d1d25d720ea9cb, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/62fde375e11b4b01932836e6363c7080] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp, totalSize=48.8 K 2024-11-18T20:41:44,001 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d13b261fa2e4fa58fd5b457bc82ccda, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1731962496335 2024-11-18T20:41:44,001 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bd21bd512254cb79eac24ff07a2b940, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1731962497112 2024-11-18T20:41:44,002 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting a1568e7022304a13b3d1d25d720ea9cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=876, earliestPutTs=1731962499463 2024-11-18T20:41:44,002 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 62fde375e11b4b01932836e6363c7080, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=886, earliestPutTs=1731962500112 2024-11-18T20:41:44,014 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#C#compaction#193 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:44,014 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/23ca7f7158834a4ab72fd589cfc9d923 is 50, key is test_row_0/C:col10/1731962500112/Put/seqid=0 2024-11-18T20:41:44,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742059_1235 (size=13221) 2024-11-18T20:41:44,050 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59378c222c2195896b2cd9530f63ae92#A#compaction#192 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:44,050 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/16fa2c06e63a4f478eb88ff7d31728cd is 175, key is test_row_0/A:col10/1731962500112/Put/seqid=0 2024-11-18T20:41:44,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742060_1236 (size=32141) 2024-11-18T20:41:44,435 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/23ca7f7158834a4ab72fd589cfc9d923 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/23ca7f7158834a4ab72fd589cfc9d923 2024-11-18T20:41:44,442 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/C of 59378c222c2195896b2cd9530f63ae92 into 23ca7f7158834a4ab72fd589cfc9d923(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:44,442 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:44,442 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/C, priority=12, startTime=1731962503563; duration=0sec 2024-11-18T20:41:44,442 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:44,442 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:C 2024-11-18T20:41:44,464 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/16fa2c06e63a4f478eb88ff7d31728cd as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/16fa2c06e63a4f478eb88ff7d31728cd 2024-11-18T20:41:44,471 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 59378c222c2195896b2cd9530f63ae92/A of 59378c222c2195896b2cd9530f63ae92 into 16fa2c06e63a4f478eb88ff7d31728cd(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:44,471 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59378c222c2195896b2cd9530f63ae92: 2024-11-18T20:41:44,471 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., storeName=59378c222c2195896b2cd9530f63ae92/A, priority=12, startTime=1731962503563; duration=0sec 2024-11-18T20:41:44,471 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:44,471 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59378c222c2195896b2cd9530f63ae92:A 2024-11-18T20:41:51,090 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-18T20:41:51,090 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:41:51,090 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:51,090 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:51,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:51,118 INFO [Thread-643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-643. 2024-11-18T20:41:51,118 DEBUG [Thread-643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:41:51,118 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:51,118 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:51,119 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:51,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T20:41:51,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-18T20:41:51,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 302 2024-11-18T20:41:51,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 266 2024-11-18T20:41:51,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 194 2024-11-18T20:41:51,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-18T20:41:51,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T20:41:51,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2420 2024-11-18T20:41:51,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2418 2024-11-18T20:41:51,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T20:41:51,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1082 2024-11-18T20:41:51,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3246 rows 2024-11-18T20:41:51,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1086 2024-11-18T20:41:51,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3257 rows 2024-11-18T20:41:51,119 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:41:51,119 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:41:51,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:51,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:51,120 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:41:51,120 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T20:41:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T20:41:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T20:41:51,124 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962511124"}]},"ts":"1731962511124"} 2024-11-18T20:41:51,126 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T20:41:51,126 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T20:41:51,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:41:51,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59378c222c2195896b2cd9530f63ae92, UNASSIGN}] 2024-11-18T20:41:51,129 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59378c222c2195896b2cd9530f63ae92, UNASSIGN 2024-11-18T20:41:51,130 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=59378c222c2195896b2cd9530f63ae92, regionState=CLOSING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:51,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=59378c222c2195896b2cd9530f63ae92, UNASSIGN because future has completed 2024-11-18T20:41:51,132 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:41:51,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:41:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T20:41:51,285 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:51,286 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:41:51,286 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 59378c222c2195896b2cd9530f63ae92, disabling compactions & flushes 2024-11-18T20:41:51,286 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:51,286 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:51,286 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. after waiting 0 ms 2024-11-18T20:41:51,286 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:51,286 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 59378c222c2195896b2cd9530f63ae92 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-18T20:41:51,286 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=A 2024-11-18T20:41:51,286 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:51,286 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=B 2024-11-18T20:41:51,286 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:51,287 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 59378c222c2195896b2cd9530f63ae92, store=C 2024-11-18T20:41:51,287 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:51,295 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111857270198a5614c9a969144daaaab4e21_59378c222c2195896b2cd9530f63ae92 is 50, key is test_row_0/A:col10/1731962511088/Put/seqid=0 2024-11-18T20:41:51,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742061_1237 (size=9914) 2024-11-18T20:41:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T20:41:51,700 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:51,708 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111857270198a5614c9a969144daaaab4e21_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111857270198a5614c9a969144daaaab4e21_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:51,710 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/2b47c083879f456bb8acf7d5bc891cee, store: [table=TestAcidGuarantees family=A region=59378c222c2195896b2cd9530f63ae92] 2024-11-18T20:41:51,711 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/2b47c083879f456bb8acf7d5bc891cee is 175, key is test_row_0/A:col10/1731962511088/Put/seqid=0 2024-11-18T20:41:51,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742062_1238 (size=22561) 2024-11-18T20:41:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T20:41:52,116 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=894, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/2b47c083879f456bb8acf7d5bc891cee 2024-11-18T20:41:52,124 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/3b295d8ac5f64851a86928f20dbaf9dc is 50, key is test_row_0/B:col10/1731962511088/Put/seqid=0 2024-11-18T20:41:52,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742063_1239 (size=9857) 2024-11-18T20:41:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T20:41:52,529 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=894 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/3b295d8ac5f64851a86928f20dbaf9dc 2024-11-18T20:41:52,539 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/61007428280a421f946a1eedc431207e is 50, key is test_row_0/C:col10/1731962511088/Put/seqid=0 2024-11-18T20:41:52,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742064_1240 (size=9857) 2024-11-18T20:41:52,945 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=894 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/61007428280a421f946a1eedc431207e 2024-11-18T20:41:52,951 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/A/2b47c083879f456bb8acf7d5bc891cee as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/2b47c083879f456bb8acf7d5bc891cee 2024-11-18T20:41:52,956 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/2b47c083879f456bb8acf7d5bc891cee, entries=100, sequenceid=894, filesize=22.0 K 2024-11-18T20:41:52,957 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/B/3b295d8ac5f64851a86928f20dbaf9dc as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3b295d8ac5f64851a86928f20dbaf9dc 2024-11-18T20:41:52,962 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3b295d8ac5f64851a86928f20dbaf9dc, entries=100, sequenceid=894, filesize=9.6 K 2024-11-18T20:41:52,963 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/.tmp/C/61007428280a421f946a1eedc431207e as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/61007428280a421f946a1eedc431207e 2024-11-18T20:41:52,967 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/61007428280a421f946a1eedc431207e, entries=100, sequenceid=894, filesize=9.6 K 2024-11-18T20:41:52,968 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 59378c222c2195896b2cd9530f63ae92 in 1682ms, sequenceid=894, compaction requested=false 2024-11-18T20:41:52,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/35c9ae9b2533484098bfe6f5770aebaf, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/abc70358821543959318894ffe9156a9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/7125a11bddb6470394e53ab2adb065f6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/d32f35e5980f47ec88b01cb26462cbab, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/08b3926ae5fc4cdc8d186019c0677437, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/01425a5ab96c4827a1724e84e44f6f55, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/5056d12225e54da9addc3e5480586be3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/77c7f68245c14632a750606453858079, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/15839cad34684b379deb1e4d299bf0b6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/9e035c9f99284f37b1e356e2723e3c3c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/01382eff060547f1864cbbe807b29e9b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/ef3e9905e7e348efac83ecf845c13be9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/fba1604ec236457799be9a2c20b6b147, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/8ce627136e99499e9950820f3183f1c6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/243ead474ab04f8f94b0b1eef72c51f7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/cf00735208c64a95a301aea93681e7de, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/93a7fb41187e420e9eed1fbc420c6b09, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/d201ac14855540e4992b9c488192149d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/7ee3f3361872420e8f3749d884c733ab, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/cbb01ef29708433b9fcb1c2ca05a3a9b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/107bfe04ae954b27925f3cc615272607, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/767fb9d23c224f30a4b4d42b0b224111, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/2bdb2404495b4bffa45737a37b70a8f0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/5b256d7d96654451a1fbcb7cbe0f524f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/9cced5c9702a4a069865b2b0c53f4a1e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/0ec9319c8adb4a6ebd7ba05574c31c7f] to archive 2024-11-18T20:41:52,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:41:52,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/35c9ae9b2533484098bfe6f5770aebaf to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/35c9ae9b2533484098bfe6f5770aebaf 2024-11-18T20:41:52,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/abc70358821543959318894ffe9156a9 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/abc70358821543959318894ffe9156a9 2024-11-18T20:41:52,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/7125a11bddb6470394e53ab2adb065f6 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/7125a11bddb6470394e53ab2adb065f6 2024-11-18T20:41:52,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/d32f35e5980f47ec88b01cb26462cbab to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/d32f35e5980f47ec88b01cb26462cbab 2024-11-18T20:41:52,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/08b3926ae5fc4cdc8d186019c0677437 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/08b3926ae5fc4cdc8d186019c0677437 2024-11-18T20:41:52,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/01425a5ab96c4827a1724e84e44f6f55 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/01425a5ab96c4827a1724e84e44f6f55 2024-11-18T20:41:52,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/5056d12225e54da9addc3e5480586be3 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/5056d12225e54da9addc3e5480586be3 2024-11-18T20:41:52,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/77c7f68245c14632a750606453858079 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/77c7f68245c14632a750606453858079 2024-11-18T20:41:52,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/15839cad34684b379deb1e4d299bf0b6 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/15839cad34684b379deb1e4d299bf0b6 2024-11-18T20:41:52,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/9e035c9f99284f37b1e356e2723e3c3c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/9e035c9f99284f37b1e356e2723e3c3c 2024-11-18T20:41:52,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/01382eff060547f1864cbbe807b29e9b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/01382eff060547f1864cbbe807b29e9b 2024-11-18T20:41:52,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/ef3e9905e7e348efac83ecf845c13be9 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/ef3e9905e7e348efac83ecf845c13be9 2024-11-18T20:41:52,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/fba1604ec236457799be9a2c20b6b147 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/fba1604ec236457799be9a2c20b6b147 2024-11-18T20:41:52,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/8ce627136e99499e9950820f3183f1c6 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/8ce627136e99499e9950820f3183f1c6 2024-11-18T20:41:52,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/243ead474ab04f8f94b0b1eef72c51f7 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/243ead474ab04f8f94b0b1eef72c51f7 2024-11-18T20:41:52,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/cf00735208c64a95a301aea93681e7de to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/cf00735208c64a95a301aea93681e7de 2024-11-18T20:41:52,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/93a7fb41187e420e9eed1fbc420c6b09 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/93a7fb41187e420e9eed1fbc420c6b09 2024-11-18T20:41:52,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/d201ac14855540e4992b9c488192149d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/d201ac14855540e4992b9c488192149d 2024-11-18T20:41:52,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/7ee3f3361872420e8f3749d884c733ab to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/7ee3f3361872420e8f3749d884c733ab 2024-11-18T20:41:52,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/cbb01ef29708433b9fcb1c2ca05a3a9b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/cbb01ef29708433b9fcb1c2ca05a3a9b 2024-11-18T20:41:52,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/107bfe04ae954b27925f3cc615272607 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/107bfe04ae954b27925f3cc615272607 2024-11-18T20:41:52,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/767fb9d23c224f30a4b4d42b0b224111 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/767fb9d23c224f30a4b4d42b0b224111 2024-11-18T20:41:53,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/2bdb2404495b4bffa45737a37b70a8f0 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/2bdb2404495b4bffa45737a37b70a8f0 2024-11-18T20:41:53,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/5b256d7d96654451a1fbcb7cbe0f524f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/5b256d7d96654451a1fbcb7cbe0f524f 2024-11-18T20:41:53,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/9cced5c9702a4a069865b2b0c53f4a1e to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/9cced5c9702a4a069865b2b0c53f4a1e 2024-11-18T20:41:53,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/0ec9319c8adb4a6ebd7ba05574c31c7f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/0ec9319c8adb4a6ebd7ba05574c31c7f 2024-11-18T20:41:53,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/29d8933ecd6f46f98fbb442545cbbfd2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5ff216eca128476b923e857616a0cd1a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/4647cae4ae6b46a491138a6087d759ae, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3873d2922e4d4e7b9f476af5ff04ef33, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/882e776eb9284b148a29faa13e4a9532, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/a4992e163d514978a4c34a2a8038ecf3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/696a4a535c184532ad70f2ad59a65006, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5497a1e1b95f48478c3127bcf5c96cf2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5f2d58342095491ab5ab270c2a323970, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/249fb47620c745719830ce116fcdac1f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/4a4a6121f96e4e8b964acde6b65cdf29, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/08d3ae9f540f4c31ba6f5ccc85a3e33b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/482407928e044424a82f7e14ebeff7e7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/b31a5c5b3faa41c0a7b000ef57798ca1, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/201614f617e8470b8e6ec5309242d92c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/b40d5605a4a448b6969d4072cd3cbf04, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/2d8d8dfe2810420d83fe9e38cd2b1313, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5c2f59c9aff84a41b18dc17f8d97cfdb, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/217dc02ab83442ceb1bdf9ceda068fe2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3692231269f1489dbd5ae45ee027d15b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/f4d8be1a8e72473bad4502526889955f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3d35953bb43443d0bf9138823c71b83e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/206abc8da02245c3b28d2e15dcdc1d5b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/c2937c47d1234eaab0b9c1187fbed3bf, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/56bca814c25e4bf596627301e517c5c0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/84afaa1e732343fa9e4aad617d715356, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/9c2f6b6505dd4c578602d93d8ea342bb] to archive 2024-11-18T20:41:53,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:41:53,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/29d8933ecd6f46f98fbb442545cbbfd2 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/29d8933ecd6f46f98fbb442545cbbfd2 2024-11-18T20:41:53,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5ff216eca128476b923e857616a0cd1a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5ff216eca128476b923e857616a0cd1a 2024-11-18T20:41:53,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/4647cae4ae6b46a491138a6087d759ae to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/4647cae4ae6b46a491138a6087d759ae 2024-11-18T20:41:53,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3873d2922e4d4e7b9f476af5ff04ef33 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3873d2922e4d4e7b9f476af5ff04ef33 2024-11-18T20:41:53,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/882e776eb9284b148a29faa13e4a9532 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/882e776eb9284b148a29faa13e4a9532 2024-11-18T20:41:53,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/a4992e163d514978a4c34a2a8038ecf3 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/a4992e163d514978a4c34a2a8038ecf3 2024-11-18T20:41:53,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/696a4a535c184532ad70f2ad59a65006 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/696a4a535c184532ad70f2ad59a65006 2024-11-18T20:41:53,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5497a1e1b95f48478c3127bcf5c96cf2 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5497a1e1b95f48478c3127bcf5c96cf2 2024-11-18T20:41:53,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5f2d58342095491ab5ab270c2a323970 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5f2d58342095491ab5ab270c2a323970 2024-11-18T20:41:53,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/249fb47620c745719830ce116fcdac1f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/249fb47620c745719830ce116fcdac1f 2024-11-18T20:41:53,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/4a4a6121f96e4e8b964acde6b65cdf29 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/4a4a6121f96e4e8b964acde6b65cdf29 2024-11-18T20:41:53,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/08d3ae9f540f4c31ba6f5ccc85a3e33b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/08d3ae9f540f4c31ba6f5ccc85a3e33b 2024-11-18T20:41:53,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/482407928e044424a82f7e14ebeff7e7 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/482407928e044424a82f7e14ebeff7e7 2024-11-18T20:41:53,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/b31a5c5b3faa41c0a7b000ef57798ca1 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/b31a5c5b3faa41c0a7b000ef57798ca1 2024-11-18T20:41:53,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/201614f617e8470b8e6ec5309242d92c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/201614f617e8470b8e6ec5309242d92c 2024-11-18T20:41:53,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/b40d5605a4a448b6969d4072cd3cbf04 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/b40d5605a4a448b6969d4072cd3cbf04 2024-11-18T20:41:53,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/2d8d8dfe2810420d83fe9e38cd2b1313 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/2d8d8dfe2810420d83fe9e38cd2b1313 2024-11-18T20:41:53,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5c2f59c9aff84a41b18dc17f8d97cfdb to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/5c2f59c9aff84a41b18dc17f8d97cfdb 2024-11-18T20:41:53,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/217dc02ab83442ceb1bdf9ceda068fe2 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/217dc02ab83442ceb1bdf9ceda068fe2 2024-11-18T20:41:53,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3692231269f1489dbd5ae45ee027d15b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3692231269f1489dbd5ae45ee027d15b 2024-11-18T20:41:53,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/f4d8be1a8e72473bad4502526889955f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/f4d8be1a8e72473bad4502526889955f 2024-11-18T20:41:53,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3d35953bb43443d0bf9138823c71b83e to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3d35953bb43443d0bf9138823c71b83e 2024-11-18T20:41:53,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/206abc8da02245c3b28d2e15dcdc1d5b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/206abc8da02245c3b28d2e15dcdc1d5b 2024-11-18T20:41:53,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/c2937c47d1234eaab0b9c1187fbed3bf to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/c2937c47d1234eaab0b9c1187fbed3bf 2024-11-18T20:41:53,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/56bca814c25e4bf596627301e517c5c0 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/56bca814c25e4bf596627301e517c5c0 2024-11-18T20:41:53,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/84afaa1e732343fa9e4aad617d715356 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/84afaa1e732343fa9e4aad617d715356 2024-11-18T20:41:53,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/9c2f6b6505dd4c578602d93d8ea342bb to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/9c2f6b6505dd4c578602d93d8ea342bb 2024-11-18T20:41:53,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/a7b0195f582c4e0c9851cc6433716689, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/1cb0368073a8456eac2ecfc4e7f9d648, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/54193b2804d74d2b8964b52382458087, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/1c768011f617403a9acf28845fda59e1, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/14c84cf58c3b47c3a65ddf0aa326c835, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/b9d183d4e8894d95a8f6de577c75eddf, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/8751fc3350e44ef38f145e74b94543a7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/5ec232c7d4544a4b80dbe546f586591a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/0ef092ffdea048af81b7a713ecd80428, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/aa66ed1ab38a461a905cade49cf375cc, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/f777375e05b346079c3adea3d182443e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/e2d5449ab1be4006b2ab6fcd22cf84f9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/cf14aa850e084555bce2c822a037561d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/1ed1663a31b5458eb369e71a8cca1728, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/7a3a081eeb574ca39e04b7486c6182e6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/eae20893234c4468a9b3f8cc98252f8f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/37bd3b8420bf45ffb5035a566db9367e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/c9ba20a8e8a84cb9807c4668b6c7d9df, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/f42019f27be4412f8ffce37de51855c5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/03a81984b0f04811b4b8ae1902f8029f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/193a4790063348b69f71d7aea1951bee, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/3d6b1bd21145471dbf85bda9ee61efd4, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/6d13b261fa2e4fa58fd5b457bc82ccda, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/61c279914aad49cba0bf46a4ba0395fc, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/4bd21bd512254cb79eac24ff07a2b940, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/a1568e7022304a13b3d1d25d720ea9cb, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/62fde375e11b4b01932836e6363c7080] to archive 2024-11-18T20:41:53,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:41:53,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/a7b0195f582c4e0c9851cc6433716689 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/a7b0195f582c4e0c9851cc6433716689 2024-11-18T20:41:53,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/1cb0368073a8456eac2ecfc4e7f9d648 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/1cb0368073a8456eac2ecfc4e7f9d648 2024-11-18T20:41:53,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/54193b2804d74d2b8964b52382458087 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/54193b2804d74d2b8964b52382458087 2024-11-18T20:41:53,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/1c768011f617403a9acf28845fda59e1 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/1c768011f617403a9acf28845fda59e1 2024-11-18T20:41:53,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/14c84cf58c3b47c3a65ddf0aa326c835 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/14c84cf58c3b47c3a65ddf0aa326c835 2024-11-18T20:41:53,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/b9d183d4e8894d95a8f6de577c75eddf to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/b9d183d4e8894d95a8f6de577c75eddf 2024-11-18T20:41:53,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/8751fc3350e44ef38f145e74b94543a7 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/8751fc3350e44ef38f145e74b94543a7 2024-11-18T20:41:53,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/5ec232c7d4544a4b80dbe546f586591a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/5ec232c7d4544a4b80dbe546f586591a 2024-11-18T20:41:53,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/0ef092ffdea048af81b7a713ecd80428 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/0ef092ffdea048af81b7a713ecd80428 2024-11-18T20:41:53,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/aa66ed1ab38a461a905cade49cf375cc to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/aa66ed1ab38a461a905cade49cf375cc 2024-11-18T20:41:53,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/f777375e05b346079c3adea3d182443e to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/f777375e05b346079c3adea3d182443e 2024-11-18T20:41:53,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/e2d5449ab1be4006b2ab6fcd22cf84f9 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/e2d5449ab1be4006b2ab6fcd22cf84f9 2024-11-18T20:41:53,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/cf14aa850e084555bce2c822a037561d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/cf14aa850e084555bce2c822a037561d 2024-11-18T20:41:53,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/1ed1663a31b5458eb369e71a8cca1728 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/1ed1663a31b5458eb369e71a8cca1728 2024-11-18T20:41:53,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/7a3a081eeb574ca39e04b7486c6182e6 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/7a3a081eeb574ca39e04b7486c6182e6 2024-11-18T20:41:53,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/eae20893234c4468a9b3f8cc98252f8f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/eae20893234c4468a9b3f8cc98252f8f 2024-11-18T20:41:53,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/37bd3b8420bf45ffb5035a566db9367e to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/37bd3b8420bf45ffb5035a566db9367e 2024-11-18T20:41:53,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/c9ba20a8e8a84cb9807c4668b6c7d9df to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/c9ba20a8e8a84cb9807c4668b6c7d9df 2024-11-18T20:41:53,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/f42019f27be4412f8ffce37de51855c5 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/f42019f27be4412f8ffce37de51855c5 2024-11-18T20:41:53,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/03a81984b0f04811b4b8ae1902f8029f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/03a81984b0f04811b4b8ae1902f8029f 2024-11-18T20:41:53,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/193a4790063348b69f71d7aea1951bee to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/193a4790063348b69f71d7aea1951bee 2024-11-18T20:41:53,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/3d6b1bd21145471dbf85bda9ee61efd4 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/3d6b1bd21145471dbf85bda9ee61efd4 2024-11-18T20:41:53,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/6d13b261fa2e4fa58fd5b457bc82ccda to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/6d13b261fa2e4fa58fd5b457bc82ccda 2024-11-18T20:41:53,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/61c279914aad49cba0bf46a4ba0395fc to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/61c279914aad49cba0bf46a4ba0395fc 2024-11-18T20:41:53,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/4bd21bd512254cb79eac24ff07a2b940 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/4bd21bd512254cb79eac24ff07a2b940 2024-11-18T20:41:53,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/a1568e7022304a13b3d1d25d720ea9cb to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/a1568e7022304a13b3d1d25d720ea9cb 2024-11-18T20:41:53,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/62fde375e11b4b01932836e6363c7080 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/62fde375e11b4b01932836e6363c7080 2024-11-18T20:41:53,078 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/recovered.edits/897.seqid, newMaxSeqId=897, maxSeqId=4 2024-11-18T20:41:53,078 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. 2024-11-18T20:41:53,078 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 59378c222c2195896b2cd9530f63ae92: Waiting for close lock at 1731962511286Running coprocessor pre-close hooks at 1731962511286Disabling compacts and flushes for region at 1731962511286Disabling writes for close at 1731962511286Obtaining lock to block concurrent updates at 1731962511286Preparing flush snapshotting stores in 59378c222c2195896b2cd9530f63ae92 at 1731962511286Finished memstore snapshotting TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731962511287 (+1 ms)Flushing stores of TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92. at 1731962511287Flushing 59378c222c2195896b2cd9530f63ae92/A: creating writer at 1731962511287Flushing mob file 59378c222c2195896b2cd9530f63ae92/A: appending metadata at 1731962511295 (+8 ms)Flushing mob file 59378c222c2195896b2cd9530f63ae92/A: closing flushed file at 1731962511295Flushing 59378c222c2195896b2cd9530f63ae92/A: appending metadata at 1731962511710 (+415 ms)Flushing 59378c222c2195896b2cd9530f63ae92/A: closing flushed file at 1731962511710Flushing 59378c222c2195896b2cd9530f63ae92/B: creating writer at 1731962512121 (+411 ms)Flushing 59378c222c2195896b2cd9530f63ae92/B: appending metadata at 1731962512124 (+3 ms)Flushing 59378c222c2195896b2cd9530f63ae92/B: closing flushed file at 1731962512124Flushing 59378c222c2195896b2cd9530f63ae92/C: creating writer at 1731962512535 (+411 ms)Flushing 59378c222c2195896b2cd9530f63ae92/C: appending metadata at 1731962512538 (+3 ms)Flushing 59378c222c2195896b2cd9530f63ae92/C: closing flushed file at 1731962512539 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38076e6b: reopening flushed file at 1731962512950 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56390da8: reopening flushed file at 1731962512956 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@537832d5: reopening flushed file at 1731962512962 (+6 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 59378c222c2195896b2cd9530f63ae92 in 1682ms, sequenceid=894, compaction requested=false at 1731962512968 (+6 ms)Writing region close event to WAL at 1731962513074 (+106 ms)Running coprocessor post-close hooks at 1731962513078 (+4 ms)Closed at 1731962513078 2024-11-18T20:41:53,080 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,081 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=59378c222c2195896b2cd9530f63ae92, regionState=CLOSED 2024-11-18T20:41:53,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 because future has completed 2024-11-18T20:41:53,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-18T20:41:53,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 59378c222c2195896b2cd9530f63ae92, server=c0a89b2656d4,45393,1731962440107 in 1.9510 sec 2024-11-18T20:41:53,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-18T20:41:53,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59378c222c2195896b2cd9530f63ae92, UNASSIGN in 1.9560 sec 2024-11-18T20:41:53,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-18T20:41:53,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9600 sec 2024-11-18T20:41:53,089 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962513089"}]},"ts":"1731962513089"} 2024-11-18T20:41:53,091 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T20:41:53,091 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T20:41:53,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9720 sec 2024-11-18T20:41:53,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T20:41:53,268 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:53,269 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T20:41:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:41:53,272 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:41:53,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T20:41:53,273 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:41:53,280 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,283 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A, FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B, FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C, FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/recovered.edits] 2024-11-18T20:41:53,287 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/16fa2c06e63a4f478eb88ff7d31728cd to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/16fa2c06e63a4f478eb88ff7d31728cd 2024-11-18T20:41:53,289 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/2b47c083879f456bb8acf7d5bc891cee to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/A/2b47c083879f456bb8acf7d5bc891cee 2024-11-18T20:41:53,293 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/2f234d63e7e64639a9ec159326699a77 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/2f234d63e7e64639a9ec159326699a77 2024-11-18T20:41:53,294 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3b295d8ac5f64851a86928f20dbaf9dc to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/B/3b295d8ac5f64851a86928f20dbaf9dc 2024-11-18T20:41:53,296 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/23ca7f7158834a4ab72fd589cfc9d923 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/23ca7f7158834a4ab72fd589cfc9d923 2024-11-18T20:41:53,298 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/61007428280a421f946a1eedc431207e to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/C/61007428280a421f946a1eedc431207e 2024-11-18T20:41:53,300 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/recovered.edits/897.seqid to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92/recovered.edits/897.seqid 2024-11-18T20:41:53,300 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,300 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T20:41:53,301 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T20:41:53,301 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-18T20:41:53,304 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111800e8c2ee9d434808aaf29b8b49cdaa24_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111800e8c2ee9d434808aaf29b8b49cdaa24_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,305 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118031fb51c265e403dbf26f2f4ad9e410f_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118031fb51c265e403dbf26f2f4ad9e410f_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,306 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111823546677c9e4445385b7fbb2ebab2996_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111823546677c9e4445385b7fbb2ebab2996_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,307 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182cabd711cd594239bc2f26d61bb31fa8_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182cabd711cd594239bc2f26d61bb31fa8_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,308 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118357501ed18e14d8a9741dd6820a3cf5b_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118357501ed18e14d8a9741dd6820a3cf5b_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,309 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183e2992c6d43144e78dbd9f11e2628ac0_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183e2992c6d43144e78dbd9f11e2628ac0_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,309 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111857270198a5614c9a969144daaaab4e21_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111857270198a5614c9a969144daaaab4e21_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,310 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111869d06872b3844392a9d27c0e4fe1abb0_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111869d06872b3844392a9d27c0e4fe1abb0_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,311 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186e9471de7a9e44269bd85fc591e7c2b2_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186e9471de7a9e44269bd85fc591e7c2b2_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,312 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111876e8bc666e3a4120b8ccf659ca5767a4_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111876e8bc666e3a4120b8ccf659ca5767a4_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,313 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187713ed11880547848ebceaf3b4d51456_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187713ed11880547848ebceaf3b4d51456_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,314 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111891b84e28abef4aee87bb5c20da5934b8_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111891b84e28abef4aee87bb5c20da5934b8_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,315 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189a431d7515e74fd1b2b51c2cff0ce237_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189a431d7515e74fd1b2b51c2cff0ce237_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,316 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189fb92e8c04db47488b3f7041610548c5_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189fb92e8c04db47488b3f7041610548c5_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,317 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a35f0bb7923143469a95f183ac56a0aa_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a35f0bb7923143469a95f183ac56a0aa_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,318 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a84badb7f31841548e743ada8f3769a8_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a84badb7f31841548e743ada8f3769a8_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,319 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b3bb485c857f408fb798b63ea6cc3258_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b3bb485c857f408fb798b63ea6cc3258_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,319 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bd26715fa0d545fb87b4a7300fb08d4e_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bd26715fa0d545fb87b4a7300fb08d4e_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,320 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d512d0f71105491088cdccf0c2dd5e1f_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d512d0f71105491088cdccf0c2dd5e1f_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,321 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e9de93b5a40a4b6e80153adc747f7eec_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e9de93b5a40a4b6e80153adc747f7eec_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,322 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f787e2dcd68e466181e74b2ece1900e9_59378c222c2195896b2cd9530f63ae92 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f787e2dcd68e466181e74b2ece1900e9_59378c222c2195896b2cd9530f63ae92 2024-11-18T20:41:53,323 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T20:41:53,325 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:41:53,327 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T20:41:53,330 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T20:41:53,331 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:41:53,331 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T20:41:53,331 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731962513331"}]},"ts":"9223372036854775807"} 2024-11-18T20:41:53,334 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T20:41:53,334 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 59378c222c2195896b2cd9530f63ae92, NAME => 'TestAcidGuarantees,,1731962477252.59378c222c2195896b2cd9530f63ae92.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T20:41:53,334 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T20:41:53,334 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731962513334"}]},"ts":"9223372036854775807"} 2024-11-18T20:41:53,336 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T20:41:53,337 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:41:53,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-11-18T20:41:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T20:41:53,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T20:41:53,378 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:53,389 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 243) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/cluster_b02ea204-637b-6699-1ed5-681f735b812e/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_105021149_22 at /127.0.0.1:40108 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/cluster_b02ea204-637b-6699-1ed5-681f735b812e/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=371 (was 357) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1883 (was 2917) 2024-11-18T20:41:53,400 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=371, ProcessCount=11, AvailableMemoryMB=1882 2024-11-18T20:41:53,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:41:53,402 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:41:53,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T20:41:53,404 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T20:41:53,404 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:53,404 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-18T20:41:53,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-18T20:41:53,405 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T20:41:53,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742065_1241 (size=1061) 2024-11-18T20:41:53,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-18T20:41:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-18T20:41:53,816 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093 2024-11-18T20:41:53,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742066_1242 (size=53) 2024-11-18T20:41:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-18T20:41:54,225 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:41:54,225 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9fe934dbc79ce3fabcdcf715bb36d192, disabling compactions & flushes 2024-11-18T20:41:54,225 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:54,225 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:54,225 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. after waiting 0 ms 2024-11-18T20:41:54,225 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:54,225 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:54,225 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9fe934dbc79ce3fabcdcf715bb36d192: Waiting for close lock at 1731962514225Disabling compacts and flushes for region at 1731962514225Disabling writes for close at 1731962514225Writing region close event to WAL at 1731962514225Closed at 1731962514225 2024-11-18T20:41:54,227 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T20:41:54,227 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731962514227"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731962514227"}]},"ts":"1731962514227"} 2024-11-18T20:41:54,229 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T20:41:54,230 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T20:41:54,231 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962514230"}]},"ts":"1731962514230"} 2024-11-18T20:41:54,234 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T20:41:54,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9fe934dbc79ce3fabcdcf715bb36d192, ASSIGN}] 2024-11-18T20:41:54,236 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9fe934dbc79ce3fabcdcf715bb36d192, ASSIGN 2024-11-18T20:41:54,237 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9fe934dbc79ce3fabcdcf715bb36d192, ASSIGN; state=OFFLINE, location=c0a89b2656d4,45393,1731962440107; forceNewPlan=false, retain=false 2024-11-18T20:41:54,388 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=9fe934dbc79ce3fabcdcf715bb36d192, regionState=OPENING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:54,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9fe934dbc79ce3fabcdcf715bb36d192, ASSIGN because future has completed 2024-11-18T20:41:54,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:41:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-18T20:41:54,548 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:54,548 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:41:54,549 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:54,549 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:41:54,549 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:54,549 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:54,551 INFO [StoreOpener-9fe934dbc79ce3fabcdcf715bb36d192-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:54,553 INFO [StoreOpener-9fe934dbc79ce3fabcdcf715bb36d192-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:41:54,553 INFO [StoreOpener-9fe934dbc79ce3fabcdcf715bb36d192-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9fe934dbc79ce3fabcdcf715bb36d192 columnFamilyName A 2024-11-18T20:41:54,553 DEBUG [StoreOpener-9fe934dbc79ce3fabcdcf715bb36d192-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:54,553 INFO [StoreOpener-9fe934dbc79ce3fabcdcf715bb36d192-1 {}] regionserver.HStore(327): Store=9fe934dbc79ce3fabcdcf715bb36d192/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:41:54,554 INFO [StoreOpener-9fe934dbc79ce3fabcdcf715bb36d192-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:54,555 INFO [StoreOpener-9fe934dbc79ce3fabcdcf715bb36d192-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:41:54,555 INFO [StoreOpener-9fe934dbc79ce3fabcdcf715bb36d192-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9fe934dbc79ce3fabcdcf715bb36d192 columnFamilyName B 2024-11-18T20:41:54,555 DEBUG [StoreOpener-9fe934dbc79ce3fabcdcf715bb36d192-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:54,556 INFO [StoreOpener-9fe934dbc79ce3fabcdcf715bb36d192-1 {}] regionserver.HStore(327): Store=9fe934dbc79ce3fabcdcf715bb36d192/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:41:54,556 INFO [StoreOpener-9fe934dbc79ce3fabcdcf715bb36d192-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:54,557 INFO [StoreOpener-9fe934dbc79ce3fabcdcf715bb36d192-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:41:54,557 INFO [StoreOpener-9fe934dbc79ce3fabcdcf715bb36d192-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9fe934dbc79ce3fabcdcf715bb36d192 columnFamilyName C 2024-11-18T20:41:54,557 DEBUG [StoreOpener-9fe934dbc79ce3fabcdcf715bb36d192-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:41:54,558 INFO [StoreOpener-9fe934dbc79ce3fabcdcf715bb36d192-1 {}] regionserver.HStore(327): Store=9fe934dbc79ce3fabcdcf715bb36d192/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:41:54,558 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:54,558 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:54,558 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:54,559 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:54,560 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:54,560 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:54,560 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:41:54,562 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:54,564 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:41:54,565 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 9fe934dbc79ce3fabcdcf715bb36d192; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67178623, jitterRate=0.0010394901037216187}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:41:54,565 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:54,565 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 9fe934dbc79ce3fabcdcf715bb36d192: Running coprocessor pre-open hook at 1731962514549Writing region info on filesystem at 1731962514549Initializing all the Stores at 1731962514550 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962514550Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962514551 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962514551Cleaning up temporary data from old regions at 1731962514560 (+9 ms)Running coprocessor post-open hooks at 1731962514565 (+5 ms)Region opened successfully at 1731962514565 2024-11-18T20:41:54,566 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., pid=65, masterSystemTime=1731962514543 2024-11-18T20:41:54,568 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:54,568 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:54,568 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=9fe934dbc79ce3fabcdcf715bb36d192, regionState=OPEN, openSeqNum=2, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:54,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 because future has completed 2024-11-18T20:41:54,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-18T20:41:54,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 in 180 msec 2024-11-18T20:41:54,574 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-18T20:41:54,574 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9fe934dbc79ce3fabcdcf715bb36d192, ASSIGN in 338 msec 2024-11-18T20:41:54,574 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T20:41:54,574 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962514574"}]},"ts":"1731962514574"} 2024-11-18T20:41:54,576 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T20:41:54,577 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T20:41:54,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-11-18T20:41:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-18T20:41:55,548 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:55,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c2f8f10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,550 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:55,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:55,552 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5672d125, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:55,555 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b66bc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,557 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:55,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:55,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:55,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:55,560 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:41:55,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is c0a89b2656d4,46825,1731962439484 2024-11-18T20:41:55,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@44b15475 2024-11-18T20:41:55,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:41:55,562 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:41:55,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6581b7e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:55,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:55,564 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:55,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:55,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:55,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eb561cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:55,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:55,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:55,566 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c9959c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20832dd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:55,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:55,568 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21b732cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:55,570 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:55,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c76af2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb66027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,571 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:55,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:55,572 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75bec4c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:55,573 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b466a02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b4a085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:55,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:55,574 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:55,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1feed50d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:55,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:55,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:55,575 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@488f288f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75f8f105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:55,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:55,577 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205a964d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:55,578 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771fc076, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29df300d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:55,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:55,580 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fc50d34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:55,581 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d195634, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@417b74fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,582 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:55,582 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:55,582 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a664bb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:55,583 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44df732b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0a17bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:55,585 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:55,585 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af5b11d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:55,587 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2da6fa17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb595da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:55,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:55,589 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22510e95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:55,590 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:55,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2093c1c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@262694dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,592 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:41:55,592 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:41:55,593 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:41:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:41:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:41:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519d7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:41:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:41:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:41:55,594 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:41:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1093c6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:41:55,595 DEBUG [Thread-1139 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:55,595 DEBUG [Thread-1143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:55,595 DEBUG [Thread-1136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:55,595 DEBUG [Thread-1141 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:55,595 DEBUG [Thread-1147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:55,595 DEBUG [Thread-1128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:55,596 DEBUG [Thread-1134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:55,596 DEBUG [Thread-1132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:55,596 DEBUG [Thread-1130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:55,596 DEBUG [Thread-1145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:41:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:55,597 DEBUG [Thread-1132 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:55,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:55,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:55,598 DEBUG [Thread-1143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:55,598 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:55,598 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:55,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:41:55,598 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:55,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:41:55,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:41:55,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:55,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-18T20:41:55,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:41:55,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:41:55,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:41:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:41:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T20:41:55,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:41:55,603 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:41:55,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:41:55,604 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:41:55,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:41:55,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:41:55,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:41:55,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:41:55,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:41:55,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:41:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:55,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T20:41:55,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:41:55,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:55,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:41:55,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:55,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:41:55,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:55,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/5874b82c9c7841afa3f19ebae8f452d5 is 50, key is test_row_0/A:col10/1731962515610/Put/seqid=0 2024-11-18T20:41:55,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742067_1243 (size=14341) 2024-11-18T20:41:55,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962575638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:55,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962515635, max timestamp=1731962515644Num uniques 859; 2024-11-18T20:41:55,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:41:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:55,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962515635, max timestamp=1731962515644Num uniques 859; 2024-11-18T20:41:55,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:41:55,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962515635, max timestamp=1731962515644Num uniques 859; 2024-11-18T20:41:55,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:41:55,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/5874b82c9c7841afa3f19ebae8f452d5 2024-11-18T20:41:55,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962575673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:55,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/53710f3f4bbd460f871d6dc922b95aee is 50, key is test_row_0/B:col10/1731962515610/Put/seqid=0 2024-11-18T20:41:55,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32890 deadline: 1731962575678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32874 deadline: 1731962575680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:55,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962575680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:55,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742068_1244 (size=12001) 2024-11-18T20:41:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T20:41:55,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/53710f3f4bbd460f871d6dc922b95aee 2024-11-18T20:41:55,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T20:41:55,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:55,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:41:55,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:55,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:55,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:55,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962575757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:55,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/438b93c899804f908691acfbecc471c3 is 50, key is test_row_0/C:col10/1731962515610/Put/seqid=0 2024-11-18T20:41:55,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742069_1245 (size=12001) 2024-11-18T20:41:55,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/438b93c899804f908691acfbecc471c3 2024-11-18T20:41:55,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/5874b82c9c7841afa3f19ebae8f452d5 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5874b82c9c7841afa3f19ebae8f452d5 2024-11-18T20:41:55,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962575787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:55,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32874 deadline: 1731962575798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:55,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962575798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:55,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32890 deadline: 1731962575798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:55,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:55,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5874b82c9c7841afa3f19ebae8f452d5, entries=200, sequenceid=12, filesize=14.0 K 2024-11-18T20:41:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:55,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/53710f3f4bbd460f871d6dc922b95aee as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/53710f3f4bbd460f871d6dc922b95aee 2024-11-18T20:41:55,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/53710f3f4bbd460f871d6dc922b95aee, entries=150, sequenceid=12, filesize=11.7 K 2024-11-18T20:41:55,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/438b93c899804f908691acfbecc471c3 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/438b93c899804f908691acfbecc471c3 2024-11-18T20:41:55,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/438b93c899804f908691acfbecc471c3, entries=150, sequenceid=12, filesize=11.7 K 2024-11-18T20:41:55,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 9fe934dbc79ce3fabcdcf715bb36d192 in 219ms, sequenceid=12, compaction requested=false 2024-11-18T20:41:55,831 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T20:41:55,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:41:55,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T20:41:55,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:55,914 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-18T20:41:55,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:41:55,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:55,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:41:55,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:55,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:41:55,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T20:41:55,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/d53d4c15e29f420ea289ef165cc0793f is 50, key is test_row_0/A:col10/1731962515678/Put/seqid=0 2024-11-18T20:41:55,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742070_1246 (size=12001) 2024-11-18T20:41:55,964 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/d53d4c15e29f420ea289ef165cc0793f 2024-11-18T20:41:55,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:41:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:55,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/f4294cce6a8b468ebafb5e52245f34eb is 50, key is test_row_0/B:col10/1731962515678/Put/seqid=0 2024-11-18T20:41:56,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742071_1247 (size=12001) 2024-11-18T20:41:56,009 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/f4294cce6a8b468ebafb5e52245f34eb 2024-11-18T20:41:56,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962576007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32874 deadline: 1731962576009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32890 deadline: 1731962576009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962576013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962576014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/86d199cd5c3448b690b1d1aa426640e2 is 50, key is test_row_0/C:col10/1731962515678/Put/seqid=0 2024-11-18T20:41:56,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742072_1248 (size=12001) 2024-11-18T20:41:56,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962576129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962576129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T20:41:56,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32874 deadline: 1731962576317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962576327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32890 deadline: 1731962576327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962576338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962576338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,462 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/86d199cd5c3448b690b1d1aa426640e2 2024-11-18T20:41:56,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/d53d4c15e29f420ea289ef165cc0793f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/d53d4c15e29f420ea289ef165cc0793f 2024-11-18T20:41:56,473 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/d53d4c15e29f420ea289ef165cc0793f, entries=150, sequenceid=47, filesize=11.7 K 2024-11-18T20:41:56,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/f4294cce6a8b468ebafb5e52245f34eb as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/f4294cce6a8b468ebafb5e52245f34eb 2024-11-18T20:41:56,480 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/f4294cce6a8b468ebafb5e52245f34eb, entries=150, sequenceid=47, filesize=11.7 K 2024-11-18T20:41:56,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/86d199cd5c3448b690b1d1aa426640e2 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/86d199cd5c3448b690b1d1aa426640e2 2024-11-18T20:41:56,485 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/86d199cd5c3448b690b1d1aa426640e2, entries=150, sequenceid=47, filesize=11.7 K 2024-11-18T20:41:56,487 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for 9fe934dbc79ce3fabcdcf715bb36d192 in 573ms, sequenceid=47, compaction requested=false 2024-11-18T20:41:56,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:41:56,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:56,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-18T20:41:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-18T20:41:56,490 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-18T20:41:56,490 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 885 msec 2024-11-18T20:41:56,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 893 msec 2024-11-18T20:41:56,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T20:41:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:56,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:41:56,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:56,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:41:56,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:56,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:41:56,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:56,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/3deaedc9f96b45668f7c0dba21f321b1 is 50, key is test_row_0/A:col10/1731962515989/Put/seqid=0 2024-11-18T20:41:56,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742073_1249 (size=14341) 2024-11-18T20:41:56,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962576707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962576713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T20:41:56,737 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:41:56,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:41:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-18T20:41:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T20:41:56,741 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:41:56,745 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:41:56,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:41:56,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962576818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32874 deadline: 1731962576828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962576828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962576837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32890 deadline: 1731962576838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:56,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:56,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:56,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T20:41:56,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T20:41:56,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:56,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:41:56,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:56,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:56,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:56,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962577027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:57,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962577037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:57,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T20:41:57,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T20:41:57,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:57,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:41:57,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:57,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/3deaedc9f96b45668f7c0dba21f321b1 2024-11-18T20:41:57,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/3841f76a0ec34973a99dfcdc3cef519c is 50, key is test_row_0/B:col10/1731962515989/Put/seqid=0 2024-11-18T20:41:57,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742074_1250 (size=12001) 2024-11-18T20:41:57,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/3841f76a0ec34973a99dfcdc3cef519c 2024-11-18T20:41:57,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/ea6253327d1d47658d7be106a25f056b is 50, key is test_row_0/C:col10/1731962515989/Put/seqid=0 2024-11-18T20:41:57,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742075_1251 (size=12001) 2024-11-18T20:41:57,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/ea6253327d1d47658d7be106a25f056b 2024-11-18T20:41:57,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/3deaedc9f96b45668f7c0dba21f321b1 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/3deaedc9f96b45668f7c0dba21f321b1 2024-11-18T20:41:57,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/3deaedc9f96b45668f7c0dba21f321b1, entries=200, sequenceid=61, filesize=14.0 K 2024-11-18T20:41:57,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/3841f76a0ec34973a99dfcdc3cef519c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/3841f76a0ec34973a99dfcdc3cef519c 2024-11-18T20:41:57,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T20:41:57,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:57,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:41:57,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:57,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/3841f76a0ec34973a99dfcdc3cef519c, entries=150, sequenceid=61, filesize=11.7 K 2024-11-18T20:41:57,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/ea6253327d1d47658d7be106a25f056b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/ea6253327d1d47658d7be106a25f056b 2024-11-18T20:41:57,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/ea6253327d1d47658d7be106a25f056b, entries=150, sequenceid=61, filesize=11.7 K 2024-11-18T20:41:57,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9fe934dbc79ce3fabcdcf715bb36d192 in 581ms, sequenceid=61, compaction requested=true 2024-11-18T20:41:57,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:41:57,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:41:57,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:57,231 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:57,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:41:57,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:57,231 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:57,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:41:57,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:57,232 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:57,232 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/A is initiating minor compaction (all files) 2024-11-18T20:41:57,232 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/A in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:57,233 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5874b82c9c7841afa3f19ebae8f452d5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/d53d4c15e29f420ea289ef165cc0793f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/3deaedc9f96b45668f7c0dba21f321b1] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=39.7 K 2024-11-18T20:41:57,233 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:57,233 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/B is initiating minor compaction (all files) 2024-11-18T20:41:57,233 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/B in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:57,233 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/53710f3f4bbd460f871d6dc922b95aee, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/f4294cce6a8b468ebafb5e52245f34eb, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/3841f76a0ec34973a99dfcdc3cef519c] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=35.2 K 2024-11-18T20:41:57,234 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 53710f3f4bbd460f871d6dc922b95aee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731962515609 2024-11-18T20:41:57,234 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5874b82c9c7841afa3f19ebae8f452d5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731962515607 2024-11-18T20:41:57,234 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting f4294cce6a8b468ebafb5e52245f34eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731962515664 2024-11-18T20:41:57,234 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting d53d4c15e29f420ea289ef165cc0793f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731962515664 2024-11-18T20:41:57,235 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 3841f76a0ec34973a99dfcdc3cef519c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731962515989 2024-11-18T20:41:57,235 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3deaedc9f96b45668f7c0dba21f321b1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731962515989 2024-11-18T20:41:57,249 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#A#compaction#207 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:57,249 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/ba223808bb7d4bf9b3f5c9b8eeb82236 is 50, key is test_row_0/A:col10/1731962515989/Put/seqid=0 2024-11-18T20:41:57,251 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#B#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:57,251 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/d1f1f02e23a14608b624ed3a5807525c is 50, key is test_row_0/B:col10/1731962515989/Put/seqid=0 2024-11-18T20:41:57,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742076_1252 (size=12104) 2024-11-18T20:41:57,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742077_1253 (size=12104) 2024-11-18T20:41:57,291 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/ba223808bb7d4bf9b3f5c9b8eeb82236 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/ba223808bb7d4bf9b3f5c9b8eeb82236 2024-11-18T20:41:57,310 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/A of 9fe934dbc79ce3fabcdcf715bb36d192 into ba223808bb7d4bf9b3f5c9b8eeb82236(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:57,310 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:41:57,310 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/A, priority=13, startTime=1731962517231; duration=0sec 2024-11-18T20:41:57,310 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:41:57,310 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:A 2024-11-18T20:41:57,310 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:41:57,311 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:41:57,311 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/C is initiating minor compaction (all files) 2024-11-18T20:41:57,311 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/C in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:57,311 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/438b93c899804f908691acfbecc471c3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/86d199cd5c3448b690b1d1aa426640e2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/ea6253327d1d47658d7be106a25f056b] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=35.2 K 2024-11-18T20:41:57,312 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 438b93c899804f908691acfbecc471c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731962515609 2024-11-18T20:41:57,313 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86d199cd5c3448b690b1d1aa426640e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731962515664 2024-11-18T20:41:57,313 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea6253327d1d47658d7be106a25f056b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731962515989 2024-11-18T20:41:57,314 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/d1f1f02e23a14608b624ed3a5807525c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/d1f1f02e23a14608b624ed3a5807525c 2024-11-18T20:41:57,321 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/B of 9fe934dbc79ce3fabcdcf715bb36d192 into d1f1f02e23a14608b624ed3a5807525c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:57,321 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:41:57,321 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/B, priority=13, startTime=1731962517231; duration=0sec 2024-11-18T20:41:57,321 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:57,321 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:B 2024-11-18T20:41:57,328 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#C#compaction#208 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:41:57,328 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/258d4deef73e4ad59ab915f2789f132b is 50, key is test_row_0/C:col10/1731962515989/Put/seqid=0 2024-11-18T20:41:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:57,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T20:41:57,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:41:57,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:57,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:41:57,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:57,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:41:57,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:57,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742078_1254 (size=12104) 2024-11-18T20:41:57,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/2bfcd3b96fdd4d68a022a859438278f8 is 50, key is test_row_0/A:col10/1731962517340/Put/seqid=0 2024-11-18T20:41:57,358 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/258d4deef73e4ad59ab915f2789f132b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/258d4deef73e4ad59ab915f2789f132b 2024-11-18T20:41:57,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742079_1255 (size=12001) 2024-11-18T20:41:57,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/2bfcd3b96fdd4d68a022a859438278f8 2024-11-18T20:41:57,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T20:41:57,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:57,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:41:57,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:57,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T20:41:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/5bef8d895756446baf91f8186ac45f4a is 50, key is test_row_0/B:col10/1731962517340/Put/seqid=0 2024-11-18T20:41:57,378 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/C of 9fe934dbc79ce3fabcdcf715bb36d192 into 258d4deef73e4ad59ab915f2789f132b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:41:57,378 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:41:57,378 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/C, priority=13, startTime=1731962517231; duration=0sec 2024-11-18T20:41:57,378 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:41:57,378 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:C 2024-11-18T20:41:57,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962577384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:57,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962577391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:57,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742080_1256 (size=12001) 2024-11-18T20:41:57,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962577497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:57,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962577498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:57,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T20:41:57,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:57,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:41:57,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:57,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T20:41:57,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:57,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:41:57,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:57,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:57,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962577707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:57,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:57,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962577707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:57,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:57,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/5bef8d895756446baf91f8186ac45f4a 2024-11-18T20:41:57,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/4305709d24624755af640cf6c960d95f is 50, key is test_row_0/C:col10/1731962517340/Put/seqid=0 2024-11-18T20:41:57,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742081_1257 (size=12001) 2024-11-18T20:41:57,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/4305709d24624755af640cf6c960d95f 2024-11-18T20:41:57,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T20:41:57,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:57,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:41:57,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:57,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:41:57,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/2bfcd3b96fdd4d68a022a859438278f8 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/2bfcd3b96fdd4d68a022a859438278f8 2024-11-18T20:41:57,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/2bfcd3b96fdd4d68a022a859438278f8, entries=150, sequenceid=82, filesize=11.7 K 2024-11-18T20:41:57,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/5bef8d895756446baf91f8186ac45f4a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/5bef8d895756446baf91f8186ac45f4a 2024-11-18T20:41:57,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:57,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962577848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32890 deadline: 1731962577848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:57,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:57,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/5bef8d895756446baf91f8186ac45f4a, entries=150, sequenceid=82, filesize=11.7 K 2024-11-18T20:41:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32874 deadline: 1731962577848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:57,853 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in 9fe934dbc79ce3fabcdcf715bb36d192 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:57,853 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 9fe934dbc79ce3fabcdcf715bb36d192 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:57,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/4305709d24624755af640cf6c960d95f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/4305709d24624755af640cf6c960d95f 2024-11-18T20:41:57,855 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in 9fe934dbc79ce3fabcdcf715bb36d192 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:57,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/4305709d24624755af640cf6c960d95f, entries=150, sequenceid=82, filesize=11.7 K 2024-11-18T20:41:57,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9fe934dbc79ce3fabcdcf715bb36d192 in 521ms, sequenceid=82, compaction requested=false 2024-11-18T20:41:57,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:41:57,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T20:41:57,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T20:41:57,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:57,994 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T20:41:57,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:41:57,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:57,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:41:57,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:57,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:41:57,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:41:58,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/788417b91c9d4907803eb53ac824c3d0 is 50, key is test_row_0/A:col10/1731962517387/Put/seqid=0 2024-11-18T20:41:58,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:58,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:41:58,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742082_1258 (size=12001) 2024-11-18T20:41:58,033 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/788417b91c9d4907803eb53ac824c3d0 2024-11-18T20:41:58,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/55f5835d9c8142cdad3624cafabe8fe5 is 50, key is test_row_0/B:col10/1731962517387/Put/seqid=0 2024-11-18T20:41:58,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742083_1259 (size=12001) 2024-11-18T20:41:58,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:58,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962578084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:58,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:58,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:58,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:58,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:58,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962578084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:58,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:58,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:58,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:58,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:58,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962578197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:58,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:58,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962578197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:58,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962578407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:58,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:58,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:58,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:58,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962578407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:58,453 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/55f5835d9c8142cdad3624cafabe8fe5 2024-11-18T20:41:58,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/666bc5f710e746929e7e34ccfa99a48f is 50, key is test_row_0/C:col10/1731962517387/Put/seqid=0 2024-11-18T20:41:58,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742084_1260 (size=12001) 2024-11-18T20:41:58,612 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T20:41:58,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962578717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:58,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962578717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T20:41:58,919 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/666bc5f710e746929e7e34ccfa99a48f 2024-11-18T20:41:58,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/788417b91c9d4907803eb53ac824c3d0 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/788417b91c9d4907803eb53ac824c3d0 2024-11-18T20:41:58,928 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/788417b91c9d4907803eb53ac824c3d0, entries=150, sequenceid=96, filesize=11.7 K 2024-11-18T20:41:58,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/55f5835d9c8142cdad3624cafabe8fe5 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/55f5835d9c8142cdad3624cafabe8fe5 2024-11-18T20:41:58,939 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/55f5835d9c8142cdad3624cafabe8fe5, entries=150, sequenceid=96, filesize=11.7 K 2024-11-18T20:41:58,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/666bc5f710e746929e7e34ccfa99a48f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/666bc5f710e746929e7e34ccfa99a48f 2024-11-18T20:41:58,953 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/666bc5f710e746929e7e34ccfa99a48f, entries=150, sequenceid=96, filesize=11.7 K 2024-11-18T20:41:58,954 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9fe934dbc79ce3fabcdcf715bb36d192 in 960ms, sequenceid=96, compaction requested=true 2024-11-18T20:41:58,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:41:58,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:41:58,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-18T20:41:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-18T20:41:58,962 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-18T20:41:58,962 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2130 sec 2024-11-18T20:41:58,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.2270 sec 2024-11-18T20:41:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:41:59,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-18T20:41:59,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731962518051, max timestamp=1731962519230Num uniques 859; 2024-11-18T20:41:59,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:41:59,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:41:59,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:59,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:41:59,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:59,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:41:59,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:41:59,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/418a8ad1ec18444490f2c29aca0b1cc5 is 50, key is test_row_0/A:col10/1731962518064/Put/seqid=0 2024-11-18T20:41:59,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742085_1261 (size=12001) 2024-11-18T20:41:59,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962579294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:59,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962579297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:59,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962579407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:59,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:59,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962579407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:59,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:59,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:59,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:59,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962579617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:59,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:59,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962579617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:59,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/418a8ad1ec18444490f2c29aca0b1cc5 2024-11-18T20:41:59,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/a990065f9d9d49f1b5536c09d10fdd2d is 50, key is test_row_0/B:col10/1731962518064/Put/seqid=0 2024-11-18T20:41:59,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742086_1262 (size=12001) 2024-11-18T20:41:59,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32890 deadline: 1731962579858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:59,861 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in 9fe934dbc79ce3fabcdcf715bb36d192 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:59,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:59,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962579868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:59,874 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 9fe934dbc79ce3fabcdcf715bb36d192 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:59,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32874 deadline: 1731962579877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:59,882 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in 9fe934dbc79ce3fabcdcf715bb36d192 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:41:59,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:59,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962579927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:59,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:41:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962579927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:41:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:41:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:41:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:00,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/a990065f9d9d49f1b5536c09d10fdd2d 2024-11-18T20:42:00,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/6ad902d61eff498b9aebcfabfa72db5d is 50, key is test_row_0/C:col10/1731962518064/Put/seqid=0 2024-11-18T20:42:00,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742087_1263 (size=12001) 2024-11-18T20:42:00,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962580437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:00,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962580437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:00,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/6ad902d61eff498b9aebcfabfa72db5d 2024-11-18T20:42:00,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/418a8ad1ec18444490f2c29aca0b1cc5 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/418a8ad1ec18444490f2c29aca0b1cc5 2024-11-18T20:42:00,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/418a8ad1ec18444490f2c29aca0b1cc5, entries=150, sequenceid=117, filesize=11.7 K 2024-11-18T20:42:00,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/a990065f9d9d49f1b5536c09d10fdd2d as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/a990065f9d9d49f1b5536c09d10fdd2d 2024-11-18T20:42:00,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/a990065f9d9d49f1b5536c09d10fdd2d, entries=150, sequenceid=117, filesize=11.7 K 2024-11-18T20:42:00,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/6ad902d61eff498b9aebcfabfa72db5d as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/6ad902d61eff498b9aebcfabfa72db5d 2024-11-18T20:42:00,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/6ad902d61eff498b9aebcfabfa72db5d, entries=150, sequenceid=117, filesize=11.7 K 2024-11-18T20:42:00,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~91.69 KB/93890, heapSize ~267.14 KB/273552, currentSize=87.22 KB/89310 for 9fe934dbc79ce3fabcdcf715bb36d192 in 1323ms, sequenceid=117, compaction requested=true 2024-11-18T20:42:00,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:00,558 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:42:00,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:42:00,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:00,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:42:00,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:00,558 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:42:00,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:42:00,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:00,559 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:42:00,559 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:42:00,559 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/A is initiating minor compaction (all files) 2024-11-18T20:42:00,559 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/B is initiating minor compaction (all files) 2024-11-18T20:42:00,559 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/B in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:00,559 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/A in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:00,559 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/d1f1f02e23a14608b624ed3a5807525c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/5bef8d895756446baf91f8186ac45f4a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/55f5835d9c8142cdad3624cafabe8fe5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/a990065f9d9d49f1b5536c09d10fdd2d] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=47.0 K 2024-11-18T20:42:00,559 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/ba223808bb7d4bf9b3f5c9b8eeb82236, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/2bfcd3b96fdd4d68a022a859438278f8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/788417b91c9d4907803eb53ac824c3d0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/418a8ad1ec18444490f2c29aca0b1cc5] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=47.0 K 2024-11-18T20:42:00,561 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting d1f1f02e23a14608b624ed3a5807525c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731962515989 2024-11-18T20:42:00,561 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bef8d895756446baf91f8186ac45f4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731962516698 2024-11-18T20:42:00,561 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba223808bb7d4bf9b3f5c9b8eeb82236, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731962515989 2024-11-18T20:42:00,562 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 55f5835d9c8142cdad3624cafabe8fe5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731962517378 2024-11-18T20:42:00,562 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bfcd3b96fdd4d68a022a859438278f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731962516698 2024-11-18T20:42:00,562 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting a990065f9d9d49f1b5536c09d10fdd2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731962518064 2024-11-18T20:42:00,562 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 788417b91c9d4907803eb53ac824c3d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731962517378 2024-11-18T20:42:00,564 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 418a8ad1ec18444490f2c29aca0b1cc5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731962518064 2024-11-18T20:42:00,575 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#B#compaction#218 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:00,575 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/18caf538b4d14e3b85da6243f7b21173 is 50, key is test_row_0/B:col10/1731962518064/Put/seqid=0 2024-11-18T20:42:00,579 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#A#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:00,579 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/5b1278e646ed47c88cef248fc2aba868 is 50, key is test_row_0/A:col10/1731962518064/Put/seqid=0 2024-11-18T20:42:00,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742089_1265 (size=12241) 2024-11-18T20:42:00,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742088_1264 (size=12241) 2024-11-18T20:42:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T20:42:00,897 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:00,898 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:42:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-18T20:42:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:42:00,900 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:42:00,901 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:42:00,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:42:01,007 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/18caf538b4d14e3b85da6243f7b21173 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/18caf538b4d14e3b85da6243f7b21173 2024-11-18T20:42:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:42:01,013 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/B of 9fe934dbc79ce3fabcdcf715bb36d192 into 18caf538b4d14e3b85da6243f7b21173(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:01,013 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:01,013 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/B, priority=12, startTime=1731962520558; duration=0sec 2024-11-18T20:42:01,013 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:01,013 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:B 2024-11-18T20:42:01,013 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:42:01,015 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:42:01,015 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/C is initiating minor compaction (all files) 2024-11-18T20:42:01,015 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/C in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:01,015 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/258d4deef73e4ad59ab915f2789f132b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/4305709d24624755af640cf6c960d95f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/666bc5f710e746929e7e34ccfa99a48f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/6ad902d61eff498b9aebcfabfa72db5d] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=47.0 K 2024-11-18T20:42:01,016 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 258d4deef73e4ad59ab915f2789f132b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731962515989 2024-11-18T20:42:01,016 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4305709d24624755af640cf6c960d95f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731962516698 2024-11-18T20:42:01,017 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 666bc5f710e746929e7e34ccfa99a48f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731962517378 2024-11-18T20:42:01,017 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ad902d61eff498b9aebcfabfa72db5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731962518064 2024-11-18T20:42:01,018 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/5b1278e646ed47c88cef248fc2aba868 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5b1278e646ed47c88cef248fc2aba868 2024-11-18T20:42:01,024 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/A of 9fe934dbc79ce3fabcdcf715bb36d192 into 5b1278e646ed47c88cef248fc2aba868(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:01,024 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:01,024 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/A, priority=12, startTime=1731962520557; duration=0sec 2024-11-18T20:42:01,024 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:01,024 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:A 2024-11-18T20:42:01,029 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#C#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:01,029 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/b4a75ed48dae4e4b85ed2256745be8e3 is 50, key is test_row_0/C:col10/1731962518064/Put/seqid=0 2024-11-18T20:42:01,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742090_1266 (size=12241) 2024-11-18T20:42:01,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T20:42:01,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:01,053 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T20:42:01,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:01,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:01,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:01,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:01,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:01,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:01,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/c34cd99525d04b2d9cf798fca483fab1 is 50, key is test_row_0/A:col10/1731962519284/Put/seqid=0 2024-11-18T20:42:01,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742091_1267 (size=12151) 2024-11-18T20:42:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:42:01,440 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/b4a75ed48dae4e4b85ed2256745be8e3 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/b4a75ed48dae4e4b85ed2256745be8e3 2024-11-18T20:42:01,448 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/C of 9fe934dbc79ce3fabcdcf715bb36d192 into b4a75ed48dae4e4b85ed2256745be8e3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:01,448 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:01,448 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/C, priority=12, startTime=1731962520558; duration=0sec 2024-11-18T20:42:01,448 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:01,448 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:C 2024-11-18T20:42:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:01,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:01,484 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/c34cd99525d04b2d9cf798fca483fab1 2024-11-18T20:42:01,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/c18e3752dabf45f78e2ab819ab018b8f is 50, key is test_row_0/B:col10/1731962519284/Put/seqid=0 2024-11-18T20:42:01,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742092_1268 (size=12151) 2024-11-18T20:42:01,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962581511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:01,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:01,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:01,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:01,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962581513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:01,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:01,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:01,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:42:01,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962581628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:01,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962581628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:01,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:01,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:01,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:01,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:01,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962581837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:01,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:01,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:01,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:01,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:01,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962581837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:01,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:01,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:01,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:01,899 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/c18e3752dabf45f78e2ab819ab018b8f 2024-11-18T20:42:01,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/537bc733a1f744119ba0cd89300c59a2 is 50, key is test_row_0/C:col10/1731962519284/Put/seqid=0 2024-11-18T20:42:01,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742093_1269 (size=12151) 2024-11-18T20:42:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:42:02,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962582147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:02,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962582148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:02,336 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/537bc733a1f744119ba0cd89300c59a2 2024-11-18T20:42:02,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/c34cd99525d04b2d9cf798fca483fab1 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/c34cd99525d04b2d9cf798fca483fab1 2024-11-18T20:42:02,345 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/c34cd99525d04b2d9cf798fca483fab1, entries=150, sequenceid=135, filesize=11.9 K 2024-11-18T20:42:02,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/c18e3752dabf45f78e2ab819ab018b8f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/c18e3752dabf45f78e2ab819ab018b8f 2024-11-18T20:42:02,351 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/c18e3752dabf45f78e2ab819ab018b8f, entries=150, sequenceid=135, filesize=11.9 K 2024-11-18T20:42:02,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/537bc733a1f744119ba0cd89300c59a2 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/537bc733a1f744119ba0cd89300c59a2 2024-11-18T20:42:02,358 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/537bc733a1f744119ba0cd89300c59a2, entries=150, sequenceid=135, filesize=11.9 K 2024-11-18T20:42:02,360 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9fe934dbc79ce3fabcdcf715bb36d192 in 1307ms, sequenceid=135, compaction requested=false 2024-11-18T20:42:02,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:02,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:02,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-18T20:42:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-18T20:42:02,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-18T20:42:02,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4600 sec 2024-11-18T20:42:02,366 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4670 sec 2024-11-18T20:42:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:02,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T20:42:02,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:02,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:02,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:02,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:02,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:02,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:02,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/cdffde8455f84380abddff2d3371769b is 50, key is test_row_0/A:col10/1731962522661/Put/seqid=0 2024-11-18T20:42:02,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742094_1270 (size=14541) 2024-11-18T20:42:02,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962582701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:02,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962582706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:02,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962582819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:02,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962582820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:03,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962583027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:03,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962583027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:42:03,048 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:03,048 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:42:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-18T20:42:03,051 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:42:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T20:42:03,053 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:42:03,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:42:03,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/cdffde8455f84380abddff2d3371769b 2024-11-18T20:42:03,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/0f90af7890dd402397dcab1cbb524715 is 50, key is test_row_0/B:col10/1731962522661/Put/seqid=0 2024-11-18T20:42:03,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742095_1271 (size=12151) 2024-11-18T20:42:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T20:42:03,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-18T20:42:03,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:03,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:03,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:03,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:03,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:03,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962583337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:03,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962583337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:03,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:03,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:03,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:03,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-18T20:42:03,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:03,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:03,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:03,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:03,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T20:42:03,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-18T20:42:03,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:03,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:03,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:03,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:03,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:03,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/0f90af7890dd402397dcab1cbb524715 2024-11-18T20:42:03,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/78bfe400f82b4465877f747108ce9846 is 50, key is test_row_0/C:col10/1731962522661/Put/seqid=0 2024-11-18T20:42:03,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742096_1272 (size=12151) 2024-11-18T20:42:03,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/78bfe400f82b4465877f747108ce9846 2024-11-18T20:42:03,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/cdffde8455f84380abddff2d3371769b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/cdffde8455f84380abddff2d3371769b 2024-11-18T20:42:03,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/cdffde8455f84380abddff2d3371769b, entries=200, sequenceid=154, filesize=14.2 K 2024-11-18T20:42:03,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/0f90af7890dd402397dcab1cbb524715 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/0f90af7890dd402397dcab1cbb524715 2024-11-18T20:42:03,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/0f90af7890dd402397dcab1cbb524715, entries=150, sequenceid=154, filesize=11.9 K 2024-11-18T20:42:03,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/78bfe400f82b4465877f747108ce9846 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/78bfe400f82b4465877f747108ce9846 2024-11-18T20:42:03,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/78bfe400f82b4465877f747108ce9846, entries=150, sequenceid=154, filesize=11.9 K 2024-11-18T20:42:03,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 9fe934dbc79ce3fabcdcf715bb36d192 in 956ms, sequenceid=154, compaction requested=true 2024-11-18T20:42:03,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:03,617 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:03,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:42:03,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:03,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:42:03,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:03,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:42:03,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:03,617 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:03,622 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:03,622 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/A is initiating minor compaction (all files) 2024-11-18T20:42:03,622 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/A in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:03,622 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5b1278e646ed47c88cef248fc2aba868, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/c34cd99525d04b2d9cf798fca483fab1, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/cdffde8455f84380abddff2d3371769b] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=38.0 K 2024-11-18T20:42:03,622 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:03,623 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/B is initiating minor compaction (all files) 2024-11-18T20:42:03,623 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/B in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:03,623 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/18caf538b4d14e3b85da6243f7b21173, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/c18e3752dabf45f78e2ab819ab018b8f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/0f90af7890dd402397dcab1cbb524715] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=35.7 K 2024-11-18T20:42:03,623 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b1278e646ed47c88cef248fc2aba868, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731962518064 2024-11-18T20:42:03,623 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18caf538b4d14e3b85da6243f7b21173, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731962518064 2024-11-18T20:42:03,624 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting c34cd99525d04b2d9cf798fca483fab1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731962519284 2024-11-18T20:42:03,624 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting c18e3752dabf45f78e2ab819ab018b8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731962519284 2024-11-18T20:42:03,624 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting cdffde8455f84380abddff2d3371769b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731962521510 2024-11-18T20:42:03,624 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f90af7890dd402397dcab1cbb524715, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731962521510 2024-11-18T20:42:03,644 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#B#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:03,646 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/f1dc33de433d4f91bd5be70ad8df7c27 is 50, key is test_row_0/B:col10/1731962522661/Put/seqid=0 2024-11-18T20:42:03,660 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#A#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:03,661 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/fdf37daff21240f98ebe6dc585a2ce9f is 50, key is test_row_0/A:col10/1731962522661/Put/seqid=0 2024-11-18T20:42:03,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-18T20:42:03,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:03,668 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T20:42:03,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:03,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:03,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:03,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:03,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:03,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T20:42:03,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/989472011c184cdca09fe3cf53fb538f is 50, key is test_row_0/A:col10/1731962522682/Put/seqid=0 2024-11-18T20:42:03,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742097_1273 (size=12493) 2024-11-18T20:42:03,725 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/f1dc33de433d4f91bd5be70ad8df7c27 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/f1dc33de433d4f91bd5be70ad8df7c27 2024-11-18T20:42:03,731 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/B of 9fe934dbc79ce3fabcdcf715bb36d192 into f1dc33de433d4f91bd5be70ad8df7c27(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:03,731 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:03,731 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/B, priority=13, startTime=1731962523617; duration=0sec 2024-11-18T20:42:03,731 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:03,731 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:B 2024-11-18T20:42:03,731 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:03,733 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:03,733 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/C is initiating minor compaction (all files) 2024-11-18T20:42:03,733 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/C in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:03,733 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/b4a75ed48dae4e4b85ed2256745be8e3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/537bc733a1f744119ba0cd89300c59a2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/78bfe400f82b4465877f747108ce9846] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=35.7 K 2024-11-18T20:42:03,733 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4a75ed48dae4e4b85ed2256745be8e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731962518064 2024-11-18T20:42:03,734 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 537bc733a1f744119ba0cd89300c59a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731962519284 2024-11-18T20:42:03,734 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78bfe400f82b4465877f747108ce9846, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731962521510 2024-11-18T20:42:03,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742098_1274 (size=12493) 2024-11-18T20:42:03,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742099_1275 (size=12151) 2024-11-18T20:42:03,759 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/989472011c184cdca09fe3cf53fb538f 2024-11-18T20:42:03,775 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#C#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:03,776 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/4a3e727d9f09498fb7413ad2df33bd5c is 50, key is test_row_0/C:col10/1731962522661/Put/seqid=0 2024-11-18T20:42:03,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/284e7b37477f4d1e9f3e235e9b0e4ec0 is 50, key is test_row_0/B:col10/1731962522682/Put/seqid=0 2024-11-18T20:42:03,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742100_1276 (size=12493) 2024-11-18T20:42:03,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742101_1277 (size=12151) 2024-11-18T20:42:03,849 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/4a3e727d9f09498fb7413ad2df33bd5c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/4a3e727d9f09498fb7413ad2df33bd5c 2024-11-18T20:42:03,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:03,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:03,858 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/C of 9fe934dbc79ce3fabcdcf715bb36d192 into 4a3e727d9f09498fb7413ad2df33bd5c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:03,858 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:03,859 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/C, priority=13, startTime=1731962523617; duration=0sec 2024-11-18T20:42:03,859 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:03,859 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:C 2024-11-18T20:42:03,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:03,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962583894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:03,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:03,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:03,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:03,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:03,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32890 deadline: 1731962583898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:03,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:03,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962583899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:03,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:03,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:03,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:03,905 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in 9fe934dbc79ce3fabcdcf715bb36d192 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:42:03,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:03,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:03,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:03,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962583902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:03,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32874 deadline: 1731962583928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:03,932 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in 9fe934dbc79ce3fabcdcf715bb36d192 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:42:03,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:03,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:03,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:04,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962584007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:04,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:04,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:04,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962584007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:04,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:04,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:04,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962584017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:04,171 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/fdf37daff21240f98ebe6dc585a2ce9f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/fdf37daff21240f98ebe6dc585a2ce9f 2024-11-18T20:42:04,183 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/A of 9fe934dbc79ce3fabcdcf715bb36d192 into fdf37daff21240f98ebe6dc585a2ce9f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:04,183 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:04,183 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/A, priority=13, startTime=1731962523617; duration=0sec 2024-11-18T20:42:04,184 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:04,184 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:A 2024-11-18T20:42:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T20:42:04,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962584217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:04,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962584218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:04,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:04,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962584227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:04,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:04,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:04,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:04,244 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/284e7b37477f4d1e9f3e235e9b0e4ec0 2024-11-18T20:42:04,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/a2283e6fc59b4ce496d0e378072b3909 is 50, key is test_row_0/C:col10/1731962522682/Put/seqid=0 2024-11-18T20:42:04,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742102_1278 (size=12151) 2024-11-18T20:42:04,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962584528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:04,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:04,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:04,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:04,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962584528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:04,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962584537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:04,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:04,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:04,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:04,711 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/a2283e6fc59b4ce496d0e378072b3909 2024-11-18T20:42:04,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/989472011c184cdca09fe3cf53fb538f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/989472011c184cdca09fe3cf53fb538f 2024-11-18T20:42:04,733 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/989472011c184cdca09fe3cf53fb538f, entries=150, sequenceid=169, filesize=11.9 K 2024-11-18T20:42:04,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/284e7b37477f4d1e9f3e235e9b0e4ec0 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/284e7b37477f4d1e9f3e235e9b0e4ec0 2024-11-18T20:42:04,757 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/284e7b37477f4d1e9f3e235e9b0e4ec0, entries=150, sequenceid=169, filesize=11.9 K 2024-11-18T20:42:04,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/a2283e6fc59b4ce496d0e378072b3909 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/a2283e6fc59b4ce496d0e378072b3909 2024-11-18T20:42:04,778 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/a2283e6fc59b4ce496d0e378072b3909, entries=150, sequenceid=169, filesize=11.9 K 2024-11-18T20:42:04,780 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 9fe934dbc79ce3fabcdcf715bb36d192 in 1112ms, sequenceid=169, compaction requested=false 2024-11-18T20:42:04,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:04,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:04,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-18T20:42:04,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-18T20:42:04,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-18T20:42:04,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7330 sec 2024-11-18T20:42:04,810 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.7590 sec 2024-11-18T20:42:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:05,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T20:42:05,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:05,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:05,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:05,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:05,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:05,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:05,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/37a28f5eff6f4237b9bc2ae6ee3fddbe is 50, key is test_row_0/A:col10/1731962523900/Put/seqid=0 2024-11-18T20:42:05,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742103_1279 (size=14541) 2024-11-18T20:42:05,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962585065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:05,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962585071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:05,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:05,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962585071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:05,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:05,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:05,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962585177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:05,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962585191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:05,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962585191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T20:42:05,198 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:05,198 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:42:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-18T20:42:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:42:05,201 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:42:05,201 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:42:05,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:42:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:42:05,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:42:05,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:05,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:05,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:05,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:05,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:05,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962585387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:05,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:05,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962585398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:05,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:05,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962585398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:05,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:05,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/37a28f5eff6f4237b9bc2ae6ee3fddbe 2024-11-18T20:42:05,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/e48178c2bd7b4b9fb70cdf28407a1610 is 50, key is test_row_0/B:col10/1731962523900/Put/seqid=0 2024-11-18T20:42:05,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:42:05,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:05,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:05,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:05,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:05,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:42:05,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742104_1280 (size=12151) 2024-11-18T20:42:05,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:42:05,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:05,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:05,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:05,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:05,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:05,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962585701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:05,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962585707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:05,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962585707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:05,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:42:05,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:05,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:05,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:05,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:05,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:05,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:42:05,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/e48178c2bd7b4b9fb70cdf28407a1610 2024-11-18T20:42:05,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/2c7dbf3194264ee3b0e383ff8b0e7653 is 50, key is test_row_0/C:col10/1731962523900/Put/seqid=0 2024-11-18T20:42:05,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742105_1281 (size=12151) 2024-11-18T20:42:05,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:42:05,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:05,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:05,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:05,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:05,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:06,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:42:06,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:06,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:06,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:06,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:06,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:06,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962586217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:06,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:06,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:06,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:06,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962586217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:06,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:06,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:06,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962586218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:06,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:06,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:06,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:06,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:06,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:42:06,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:06,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:06,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:06,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:06,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:42:06,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/2c7dbf3194264ee3b0e383ff8b0e7653 2024-11-18T20:42:06,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/37a28f5eff6f4237b9bc2ae6ee3fddbe as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/37a28f5eff6f4237b9bc2ae6ee3fddbe 2024-11-18T20:42:06,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/37a28f5eff6f4237b9bc2ae6ee3fddbe, entries=200, sequenceid=190, filesize=14.2 K 2024-11-18T20:42:06,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/e48178c2bd7b4b9fb70cdf28407a1610 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/e48178c2bd7b4b9fb70cdf28407a1610 2024-11-18T20:42:06,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/e48178c2bd7b4b9fb70cdf28407a1610, entries=150, sequenceid=190, filesize=11.9 K 2024-11-18T20:42:06,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/2c7dbf3194264ee3b0e383ff8b0e7653 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/2c7dbf3194264ee3b0e383ff8b0e7653 2024-11-18T20:42:06,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/2c7dbf3194264ee3b0e383ff8b0e7653, entries=150, sequenceid=190, filesize=11.9 K 2024-11-18T20:42:06,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 9fe934dbc79ce3fabcdcf715bb36d192 in 1377ms, sequenceid=190, compaction requested=true 2024-11-18T20:42:06,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:06,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:42:06,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:06,416 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:06,416 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:06,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:42:06,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:06,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:42:06,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:06,417 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:06,417 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/B is initiating minor compaction (all files) 2024-11-18T20:42:06,418 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/B in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:06,418 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/f1dc33de433d4f91bd5be70ad8df7c27, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/284e7b37477f4d1e9f3e235e9b0e4ec0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/e48178c2bd7b4b9fb70cdf28407a1610] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=35.9 K 2024-11-18T20:42:06,418 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:06,418 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/A is initiating minor compaction (all files) 2024-11-18T20:42:06,418 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/A in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:06,418 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/fdf37daff21240f98ebe6dc585a2ce9f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/989472011c184cdca09fe3cf53fb538f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/37a28f5eff6f4237b9bc2ae6ee3fddbe] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=38.3 K 2024-11-18T20:42:06,419 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting f1dc33de433d4f91bd5be70ad8df7c27, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731962521510 2024-11-18T20:42:06,420 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdf37daff21240f98ebe6dc585a2ce9f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731962521510 2024-11-18T20:42:06,420 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 989472011c184cdca09fe3cf53fb538f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731962522682 2024-11-18T20:42:06,420 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 284e7b37477f4d1e9f3e235e9b0e4ec0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731962522682 2024-11-18T20:42:06,420 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37a28f5eff6f4237b9bc2ae6ee3fddbe, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731962523887 2024-11-18T20:42:06,421 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting e48178c2bd7b4b9fb70cdf28407a1610, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731962523897 2024-11-18T20:42:06,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:42:06,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:06,433 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T20:42:06,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:06,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:06,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:06,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:06,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:06,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:06,439 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#A#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:06,440 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/bd35ac5b886c4538b496ae259a8d8b91 is 50, key is test_row_0/A:col10/1731962523900/Put/seqid=0 2024-11-18T20:42:06,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/051b7f7cde1741708cf6851f4f895a07 is 50, key is test_row_0/A:col10/1731962525068/Put/seqid=0 2024-11-18T20:42:06,451 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#B#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:06,452 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/d1d83239ca6b4b079afc76d7e6d19430 is 50, key is test_row_0/B:col10/1731962523900/Put/seqid=0 2024-11-18T20:42:06,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742107_1283 (size=12151) 2024-11-18T20:42:06,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742106_1282 (size=12595) 2024-11-18T20:42:06,502 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/bd35ac5b886c4538b496ae259a8d8b91 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/bd35ac5b886c4538b496ae259a8d8b91 2024-11-18T20:42:06,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742108_1284 (size=12595) 2024-11-18T20:42:06,519 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/A of 9fe934dbc79ce3fabcdcf715bb36d192 into bd35ac5b886c4538b496ae259a8d8b91(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:06,520 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:06,520 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/A, priority=13, startTime=1731962526416; duration=0sec 2024-11-18T20:42:06,520 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:06,520 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:A 2024-11-18T20:42:06,520 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:06,523 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/d1d83239ca6b4b079afc76d7e6d19430 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/d1d83239ca6b4b079afc76d7e6d19430 2024-11-18T20:42:06,525 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:06,526 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/C is initiating minor compaction (all files) 2024-11-18T20:42:06,526 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/C in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:06,526 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/4a3e727d9f09498fb7413ad2df33bd5c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/a2283e6fc59b4ce496d0e378072b3909, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/2c7dbf3194264ee3b0e383ff8b0e7653] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=35.9 K 2024-11-18T20:42:06,527 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a3e727d9f09498fb7413ad2df33bd5c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731962521510 2024-11-18T20:42:06,527 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2283e6fc59b4ce496d0e378072b3909, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731962522682 2024-11-18T20:42:06,528 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c7dbf3194264ee3b0e383ff8b0e7653, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731962523897 2024-11-18T20:42:06,531 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/B of 9fe934dbc79ce3fabcdcf715bb36d192 into d1d83239ca6b4b079afc76d7e6d19430(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:06,531 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:06,531 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/B, priority=13, startTime=1731962526416; duration=0sec 2024-11-18T20:42:06,531 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:06,531 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:B 2024-11-18T20:42:06,571 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#C#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:06,572 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/a1ee03b7e17d465cb0c1ea0094683972 is 50, key is test_row_0/C:col10/1731962523900/Put/seqid=0 2024-11-18T20:42:06,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742109_1285 (size=12595) 2024-11-18T20:42:06,887 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/051b7f7cde1741708cf6851f4f895a07 2024-11-18T20:42:06,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/4fcee9c0bc8a4db5a14518b506634c4f is 50, key is test_row_0/B:col10/1731962525068/Put/seqid=0 2024-11-18T20:42:06,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742110_1286 (size=12151) 2024-11-18T20:42:06,954 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/4fcee9c0bc8a4db5a14518b506634c4f 2024-11-18T20:42:06,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/09c9e58212984d6d8f79d66a25427584 is 50, key is test_row_0/C:col10/1731962525068/Put/seqid=0 2024-11-18T20:42:06,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742111_1287 (size=12151) 2024-11-18T20:42:07,029 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/a1ee03b7e17d465cb0c1ea0094683972 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/a1ee03b7e17d465cb0c1ea0094683972 2024-11-18T20:42:07,034 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/C of 9fe934dbc79ce3fabcdcf715bb36d192 into a1ee03b7e17d465cb0c1ea0094683972(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:07,034 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:07,034 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/C, priority=13, startTime=1731962526416; duration=0sec 2024-11-18T20:42:07,034 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:07,034 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:C 2024-11-18T20:42:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:07,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:07,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962587262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:07,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962587263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962587263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:42:07,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:07,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962587377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:07,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:07,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962587377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:07,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:07,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:07,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962587378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:07,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:07,395 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/09c9e58212984d6d8f79d66a25427584 2024-11-18T20:42:07,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/051b7f7cde1741708cf6851f4f895a07 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/051b7f7cde1741708cf6851f4f895a07 2024-11-18T20:42:07,407 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/051b7f7cde1741708cf6851f4f895a07, entries=150, sequenceid=204, filesize=11.9 K 2024-11-18T20:42:07,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/4fcee9c0bc8a4db5a14518b506634c4f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/4fcee9c0bc8a4db5a14518b506634c4f 2024-11-18T20:42:07,423 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/4fcee9c0bc8a4db5a14518b506634c4f, entries=150, sequenceid=204, filesize=11.9 K 2024-11-18T20:42:07,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/09c9e58212984d6d8f79d66a25427584 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/09c9e58212984d6d8f79d66a25427584 2024-11-18T20:42:07,433 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/09c9e58212984d6d8f79d66a25427584, entries=150, sequenceid=204, filesize=11.9 K 2024-11-18T20:42:07,437 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 9fe934dbc79ce3fabcdcf715bb36d192 in 1004ms, sequenceid=204, compaction requested=false 2024-11-18T20:42:07,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:07,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:07,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-18T20:42:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-18T20:42:07,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-18T20:42:07,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.2520 sec 2024-11-18T20:42:07,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2390 sec 2024-11-18T20:42:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:07,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T20:42:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:07,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/9139f84ea3904919adb68b06e32ded5b is 50, key is test_row_0/A:col10/1731962527262/Put/seqid=0 2024-11-18T20:42:07,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742112_1288 (size=14541) 2024-11-18T20:42:07,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/9139f84ea3904919adb68b06e32ded5b 2024-11-18T20:42:07,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962587606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962587606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:07,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/b48e96ed22254866896f33e333d4aaff is 50, key is test_row_0/B:col10/1731962527262/Put/seqid=0 2024-11-18T20:42:07,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962587614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:07,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742113_1289 (size=12151) 2024-11-18T20:42:07,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/b48e96ed22254866896f33e333d4aaff 2024-11-18T20:42:07,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/33a8d04053b34e2f90bd5105608da743 is 50, key is test_row_0/C:col10/1731962527262/Put/seqid=0 2024-11-18T20:42:07,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742114_1290 (size=12151) 2024-11-18T20:42:07,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:07,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962587717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:07,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:07,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962587717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:07,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:07,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:07,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962587727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:07,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:07,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:07,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962587927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:07,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962587927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:07,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:07,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:07,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:07,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962587937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:08,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/33a8d04053b34e2f90bd5105608da743 2024-11-18T20:42:08,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/9139f84ea3904919adb68b06e32ded5b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/9139f84ea3904919adb68b06e32ded5b 2024-11-18T20:42:08,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/9139f84ea3904919adb68b06e32ded5b, entries=200, sequenceid=227, filesize=14.2 K 2024-11-18T20:42:08,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/b48e96ed22254866896f33e333d4aaff as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/b48e96ed22254866896f33e333d4aaff 2024-11-18T20:42:08,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/b48e96ed22254866896f33e333d4aaff, entries=150, sequenceid=227, filesize=11.9 K 2024-11-18T20:42:08,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/33a8d04053b34e2f90bd5105608da743 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/33a8d04053b34e2f90bd5105608da743 2024-11-18T20:42:08,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/33a8d04053b34e2f90bd5105608da743, entries=150, sequenceid=227, filesize=11.9 K 2024-11-18T20:42:08,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9fe934dbc79ce3fabcdcf715bb36d192 in 526ms, sequenceid=227, compaction requested=true 2024-11-18T20:42:08,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:08,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:42:08,116 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:08,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:08,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:42:08,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:08,116 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:08,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:42:08,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:08,117 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:08,117 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/A is initiating minor compaction (all files) 2024-11-18T20:42:08,117 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/A in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:08,117 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/bd35ac5b886c4538b496ae259a8d8b91, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/051b7f7cde1741708cf6851f4f895a07, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/9139f84ea3904919adb68b06e32ded5b] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=38.4 K 2024-11-18T20:42:08,118 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:08,118 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd35ac5b886c4538b496ae259a8d8b91, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731962523897 2024-11-18T20:42:08,118 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/B is initiating minor compaction (all files) 2024-11-18T20:42:08,119 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/B in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:08,119 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/d1d83239ca6b4b079afc76d7e6d19430, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/4fcee9c0bc8a4db5a14518b506634c4f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/b48e96ed22254866896f33e333d4aaff] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=36.0 K 2024-11-18T20:42:08,119 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 051b7f7cde1741708cf6851f4f895a07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731962525060 2024-11-18T20:42:08,119 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting d1d83239ca6b4b079afc76d7e6d19430, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731962523897 2024-11-18T20:42:08,119 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9139f84ea3904919adb68b06e32ded5b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731962527259 2024-11-18T20:42:08,119 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fcee9c0bc8a4db5a14518b506634c4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731962525060 2024-11-18T20:42:08,120 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting b48e96ed22254866896f33e333d4aaff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731962527261 2024-11-18T20:42:08,144 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#A#compaction#245 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:08,145 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/2cc749f4de6448b39db904a87ccf6206 is 50, key is test_row_0/A:col10/1731962527262/Put/seqid=0 2024-11-18T20:42:08,146 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#B#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:08,147 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/4dcda4e28a284e65a4996f364b5a8bb0 is 50, key is test_row_0/B:col10/1731962527262/Put/seqid=0 2024-11-18T20:42:08,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742115_1291 (size=12697) 2024-11-18T20:42:08,162 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/2cc749f4de6448b39db904a87ccf6206 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/2cc749f4de6448b39db904a87ccf6206 2024-11-18T20:42:08,178 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/A of 9fe934dbc79ce3fabcdcf715bb36d192 into 2cc749f4de6448b39db904a87ccf6206(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:08,178 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:08,178 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/A, priority=13, startTime=1731962528116; duration=0sec 2024-11-18T20:42:08,178 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:08,178 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:A 2024-11-18T20:42:08,178 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:08,181 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:08,181 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/C is initiating minor compaction (all files) 2024-11-18T20:42:08,182 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/C in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:08,182 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/a1ee03b7e17d465cb0c1ea0094683972, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/09c9e58212984d6d8f79d66a25427584, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/33a8d04053b34e2f90bd5105608da743] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=36.0 K 2024-11-18T20:42:08,182 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1ee03b7e17d465cb0c1ea0094683972, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731962523897 2024-11-18T20:42:08,183 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09c9e58212984d6d8f79d66a25427584, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731962525060 2024-11-18T20:42:08,184 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33a8d04053b34e2f90bd5105608da743, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731962527261 2024-11-18T20:42:08,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742116_1292 (size=12697) 2024-11-18T20:42:08,236 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#C#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:08,236 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/105f91e1cc6a47e3a4b748cb5fb72e7f is 50, key is test_row_0/C:col10/1731962527262/Put/seqid=0 2024-11-18T20:42:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:08,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T20:42:08,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:08,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:08,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:08,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:08,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:08,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:08,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742117_1293 (size=12697) 2024-11-18T20:42:08,261 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/105f91e1cc6a47e3a4b748cb5fb72e7f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/105f91e1cc6a47e3a4b748cb5fb72e7f 2024-11-18T20:42:08,269 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/C of 9fe934dbc79ce3fabcdcf715bb36d192 into 105f91e1cc6a47e3a4b748cb5fb72e7f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:08,269 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:08,269 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/C, priority=13, startTime=1731962528116; duration=0sec 2024-11-18T20:42:08,269 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:08,269 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:C 2024-11-18T20:42:08,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/6c37edfceb3a4413a2f6da23b7bd8e74 is 50, key is test_row_0/A:col10/1731962527611/Put/seqid=0 2024-11-18T20:42:08,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962588288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:08,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962588296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:08,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962588296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:08,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742118_1294 (size=16931) 2024-11-18T20:42:08,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/6c37edfceb3a4413a2f6da23b7bd8e74 2024-11-18T20:42:08,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/ebc20ad99aa741c5a6e30cf5d5fd57d1 is 50, key is test_row_0/B:col10/1731962527611/Put/seqid=0 2024-11-18T20:42:08,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742119_1295 (size=12151) 2024-11-18T20:42:08,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/ebc20ad99aa741c5a6e30cf5d5fd57d1 2024-11-18T20:42:08,408 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:42:08,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962588407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:08,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962588408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:08,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962588408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:08,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/c8cbc274a6a34f52b442f0c0b411cf43 is 50, key is test_row_0/C:col10/1731962527611/Put/seqid=0 2024-11-18T20:42:08,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742120_1296 (size=12151) 2024-11-18T20:42:08,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962588619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:08,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962588619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:08,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962588629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:08,635 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/4dcda4e28a284e65a4996f364b5a8bb0 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/4dcda4e28a284e65a4996f364b5a8bb0 2024-11-18T20:42:08,651 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/B of 9fe934dbc79ce3fabcdcf715bb36d192 into 4dcda4e28a284e65a4996f364b5a8bb0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:08,651 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:08,651 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/B, priority=13, startTime=1731962528116; duration=0sec 2024-11-18T20:42:08,654 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:08,654 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:B 2024-11-18T20:42:08,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/c8cbc274a6a34f52b442f0c0b411cf43 2024-11-18T20:42:08,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/6c37edfceb3a4413a2f6da23b7bd8e74 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/6c37edfceb3a4413a2f6da23b7bd8e74 2024-11-18T20:42:08,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/6c37edfceb3a4413a2f6da23b7bd8e74, entries=250, sequenceid=242, filesize=16.5 K 2024-11-18T20:42:08,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/ebc20ad99aa741c5a6e30cf5d5fd57d1 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/ebc20ad99aa741c5a6e30cf5d5fd57d1 2024-11-18T20:42:08,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/ebc20ad99aa741c5a6e30cf5d5fd57d1, entries=150, sequenceid=242, filesize=11.9 K 2024-11-18T20:42:08,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/c8cbc274a6a34f52b442f0c0b411cf43 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/c8cbc274a6a34f52b442f0c0b411cf43 2024-11-18T20:42:08,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/c8cbc274a6a34f52b442f0c0b411cf43, entries=150, sequenceid=242, filesize=11.9 K 2024-11-18T20:42:08,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 9fe934dbc79ce3fabcdcf715bb36d192 in 637ms, sequenceid=242, compaction requested=false 2024-11-18T20:42:08,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:08,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T20:42:08,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:08,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:08,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:08,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:08,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:08,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:08,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/b3cf083785a94c2aa36fa55c98d3fee8 is 50, key is test_row_0/A:col10/1731962528274/Put/seqid=0 2024-11-18T20:42:08,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742121_1297 (size=14691) 2024-11-18T20:42:08,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/b3cf083785a94c2aa36fa55c98d3fee8 2024-11-18T20:42:08,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962588955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:08,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962588957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962588958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:08,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/2292448389474fd7888e8b361d9510b4 is 50, key is test_row_0/B:col10/1731962528274/Put/seqid=0 2024-11-18T20:42:08,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:08,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:08,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742122_1298 (size=12251) 2024-11-18T20:42:09,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962589068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:09,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962589068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:09,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962589077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:09,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:09,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:09,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:09,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962589278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:09,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:09,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962589278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:09,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:09,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:09,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:09,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:09,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:09,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962589287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:09,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:09,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:09,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:42:09,357 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:09,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:42:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-18T20:42:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T20:42:09,363 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:42:09,364 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:42:09,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:42:09,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/2292448389474fd7888e8b361d9510b4 2024-11-18T20:42:09,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/aab26774ddfa4580945f88c0dfda4112 is 50, key is test_row_0/C:col10/1731962528274/Put/seqid=0 2024-11-18T20:42:09,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742123_1299 (size=12251) 2024-11-18T20:42:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T20:42:09,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-18T20:42:09,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:09,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:09,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:09,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:09,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:09,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962589588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:09,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962589588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:09,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962589597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:09,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-18T20:42:09,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:09,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:09,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:09,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:09,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T20:42:09,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-18T20:42:09,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:09,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:09,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:09,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:09,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:09,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:09,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/aab26774ddfa4580945f88c0dfda4112 2024-11-18T20:42:09,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/b3cf083785a94c2aa36fa55c98d3fee8 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/b3cf083785a94c2aa36fa55c98d3fee8 2024-11-18T20:42:09,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/b3cf083785a94c2aa36fa55c98d3fee8, entries=200, sequenceid=264, filesize=14.3 K 2024-11-18T20:42:09,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/2292448389474fd7888e8b361d9510b4 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/2292448389474fd7888e8b361d9510b4 2024-11-18T20:42:09,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/2292448389474fd7888e8b361d9510b4, entries=150, sequenceid=264, filesize=12.0 K 2024-11-18T20:42:09,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/aab26774ddfa4580945f88c0dfda4112 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/aab26774ddfa4580945f88c0dfda4112 2024-11-18T20:42:09,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/aab26774ddfa4580945f88c0dfda4112, entries=150, sequenceid=264, filesize=12.0 K 2024-11-18T20:42:09,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 9fe934dbc79ce3fabcdcf715bb36d192 in 929ms, sequenceid=264, compaction requested=true 2024-11-18T20:42:09,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:09,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:42:09,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:09,859 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:09,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:42:09,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:09,859 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:09,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:42:09,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:09,860 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44319 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:09,860 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:09,860 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/B is initiating minor compaction (all files) 2024-11-18T20:42:09,860 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/A is initiating minor compaction (all files) 2024-11-18T20:42:09,860 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/A in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:09,860 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/2cc749f4de6448b39db904a87ccf6206, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/6c37edfceb3a4413a2f6da23b7bd8e74, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/b3cf083785a94c2aa36fa55c98d3fee8] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=43.3 K 2024-11-18T20:42:09,860 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cc749f4de6448b39db904a87ccf6206, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731962527261 2024-11-18T20:42:09,860 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/B in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:09,861 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/4dcda4e28a284e65a4996f364b5a8bb0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/ebc20ad99aa741c5a6e30cf5d5fd57d1, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/2292448389474fd7888e8b361d9510b4] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=36.2 K 2024-11-18T20:42:09,861 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c37edfceb3a4413a2f6da23b7bd8e74, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731962527598 2024-11-18T20:42:09,861 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dcda4e28a284e65a4996f364b5a8bb0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731962527261 2024-11-18T20:42:09,861 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3cf083785a94c2aa36fa55c98d3fee8, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731962528274 2024-11-18T20:42:09,861 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting ebc20ad99aa741c5a6e30cf5d5fd57d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731962527611 2024-11-18T20:42:09,862 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 2292448389474fd7888e8b361d9510b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731962528274 2024-11-18T20:42:09,871 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#B#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:09,871 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/d2ce26dcad284f6faaaa3a05e0e1e9a8 is 50, key is test_row_0/B:col10/1731962528274/Put/seqid=0 2024-11-18T20:42:09,871 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#A#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:09,872 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/5e85b61a0af44e0fa219c4c4f36e7686 is 50, key is test_row_0/A:col10/1731962528274/Put/seqid=0 2024-11-18T20:42:09,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742124_1300 (size=12899) 2024-11-18T20:42:09,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742125_1301 (size=12899) 2024-11-18T20:42:09,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-18T20:42:09,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:09,980 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T20:42:09,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:09,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:09,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:09,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:09,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:09,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:09,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T20:42:09,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/290fca8c1a8e484496225917641cb806 is 50, key is test_row_0/A:col10/1731962528954/Put/seqid=0 2024-11-18T20:42:10,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742126_1302 (size=12301) 2024-11-18T20:42:10,015 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/290fca8c1a8e484496225917641cb806 2024-11-18T20:42:10,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/7d2c96a55dc544359c115650a28dd372 is 50, key is test_row_0/B:col10/1731962528954/Put/seqid=0 2024-11-18T20:42:10,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742127_1303 (size=12301) 2024-11-18T20:42:10,065 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/7d2c96a55dc544359c115650a28dd372 2024-11-18T20:42:10,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/1fe154946a2e46db972fb2e0505af75b is 50, key is test_row_0/C:col10/1731962528954/Put/seqid=0 2024-11-18T20:42:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:10,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:10,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742128_1304 (size=12301) 2024-11-18T20:42:10,143 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/1fe154946a2e46db972fb2e0505af75b 2024-11-18T20:42:10,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/290fca8c1a8e484496225917641cb806 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/290fca8c1a8e484496225917641cb806 2024-11-18T20:42:10,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962590150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:10,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:10,162 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/290fca8c1a8e484496225917641cb806, entries=150, sequenceid=277, filesize=12.0 K 2024-11-18T20:42:10,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962590156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:10,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:10,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962590158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:10,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:10,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/7d2c96a55dc544359c115650a28dd372 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/7d2c96a55dc544359c115650a28dd372 2024-11-18T20:42:10,176 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/7d2c96a55dc544359c115650a28dd372, entries=150, sequenceid=277, filesize=12.0 K 2024-11-18T20:42:10,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/1fe154946a2e46db972fb2e0505af75b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/1fe154946a2e46db972fb2e0505af75b 2024-11-18T20:42:10,195 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/1fe154946a2e46db972fb2e0505af75b, entries=150, sequenceid=277, filesize=12.0 K 2024-11-18T20:42:10,198 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 9fe934dbc79ce3fabcdcf715bb36d192 in 218ms, sequenceid=277, compaction requested=true 2024-11-18T20:42:10,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:10,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:10,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-18T20:42:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-18T20:42:10,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-18T20:42:10,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 837 msec 2024-11-18T20:42:10,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 850 msec 2024-11-18T20:42:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:10,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962530144, max timestamp=1731962530270Num uniques 859; 2024-11-18T20:42:10,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:42:10,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962530144, max timestamp=1731962530270Num uniques 859; 2024-11-18T20:42:10,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:42:10,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962530144, max timestamp=1731962530270Num uniques 859; 2024-11-18T20:42:10,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:42:10,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=40.25 KB heapSize=106.08 KB 2024-11-18T20:42:10,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:10,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:42:10,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:10,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:42:10,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:10,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:42:10,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/64c7f396241a4585a884f253132d792d is 50, key is test_row_0/A:col10/1731962530273/Put/seqid=0 2024-11-18T20:42:10,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742129_1305 (size=14741) 2024-11-18T20:42:10,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/64c7f396241a4585a884f253132d792d 2024-11-18T20:42:10,322 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/5e85b61a0af44e0fa219c4c4f36e7686 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5e85b61a0af44e0fa219c4c4f36e7686 2024-11-18T20:42:10,329 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/d2ce26dcad284f6faaaa3a05e0e1e9a8 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/d2ce26dcad284f6faaaa3a05e0e1e9a8 2024-11-18T20:42:10,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/1f7d13ef2eee4c819bfe01fcd60998bc is 50, key is test_row_0/B:col10/1731962530273/Put/seqid=0 2024-11-18T20:42:10,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962530308, max timestamp=1731962530331Num uniques 859; 2024-11-18T20:42:10,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:42:10,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962530308, max timestamp=1731962530331Num uniques 859; 2024-11-18T20:42:10,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:42:10,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962530308, max timestamp=1731962530331Num uniques 859; 2024-11-18T20:42:10,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:42:10,336 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/A of 9fe934dbc79ce3fabcdcf715bb36d192 into 5e85b61a0af44e0fa219c4c4f36e7686(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:10,336 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:10,336 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/A, priority=13, startTime=1731962529859; duration=0sec 2024-11-18T20:42:10,336 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:10,336 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:A 2024-11-18T20:42:10,336 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:42:10,340 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:42:10,340 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/C is initiating minor compaction (all files) 2024-11-18T20:42:10,340 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/C in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:10,340 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/105f91e1cc6a47e3a4b748cb5fb72e7f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/c8cbc274a6a34f52b442f0c0b411cf43, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/aab26774ddfa4580945f88c0dfda4112, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/1fe154946a2e46db972fb2e0505af75b] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=48.2 K 2024-11-18T20:42:10,341 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 105f91e1cc6a47e3a4b748cb5fb72e7f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731962527261 2024-11-18T20:42:10,343 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8cbc274a6a34f52b442f0c0b411cf43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731962527611 2024-11-18T20:42:10,344 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/B of 9fe934dbc79ce3fabcdcf715bb36d192 into d2ce26dcad284f6faaaa3a05e0e1e9a8(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:10,344 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:10,344 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/B, priority=13, startTime=1731962529859; duration=0sec 2024-11-18T20:42:10,344 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:10,344 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:B 2024-11-18T20:42:10,344 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting aab26774ddfa4580945f88c0dfda4112, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731962528274 2024-11-18T20:42:10,345 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fe154946a2e46db972fb2e0505af75b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731962528937 2024-11-18T20:42:10,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742130_1306 (size=12301) 2024-11-18T20:42:10,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/1f7d13ef2eee4c819bfe01fcd60998bc 2024-11-18T20:42:10,359 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#C#compaction#261 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:10,359 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/d2fd99b4c3a04b4a9674f489826fecc6 is 50, key is test_row_0/C:col10/1731962528954/Put/seqid=0 2024-11-18T20:42:10,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/f76c1f30d57946489b7a5f8beff25288 is 50, key is test_row_0/C:col10/1731962530273/Put/seqid=0 2024-11-18T20:42:10,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742131_1307 (size=12983) 2024-11-18T20:42:10,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731962530403, max timestamp=1731962530427Num uniques 1018; 2024-11-18T20:42:10,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-18T20:42:10,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731962530403, max timestamp=1731962530427Num uniques 1018; 2024-11-18T20:42:10,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-18T20:42:10,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731962530403, max timestamp=1731962530427Num uniques 1018; 2024-11-18T20:42:10,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-18T20:42:10,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742132_1308 (size=12301) 2024-11-18T20:42:10,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/f76c1f30d57946489b7a5f8beff25288 2024-11-18T20:42:10,440 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/d2fd99b4c3a04b4a9674f489826fecc6 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/d2fd99b4c3a04b4a9674f489826fecc6 2024-11-18T20:42:10,453 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/C of 9fe934dbc79ce3fabcdcf715bb36d192 into d2fd99b4c3a04b4a9674f489826fecc6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:10,453 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:10,453 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/C, priority=12, startTime=1731962529859; duration=0sec 2024-11-18T20:42:10,453 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:10,453 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:C 2024-11-18T20:42:10,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/64c7f396241a4585a884f253132d792d as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/64c7f396241a4585a884f253132d792d 2024-11-18T20:42:10,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/64c7f396241a4585a884f253132d792d, entries=200, sequenceid=300, filesize=14.4 K 2024-11-18T20:42:10,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/1f7d13ef2eee4c819bfe01fcd60998bc as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/1f7d13ef2eee4c819bfe01fcd60998bc 2024-11-18T20:42:10,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/1f7d13ef2eee4c819bfe01fcd60998bc, entries=150, sequenceid=300, filesize=12.0 K 2024-11-18T20:42:10,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/f76c1f30d57946489b7a5f8beff25288 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/f76c1f30d57946489b7a5f8beff25288 2024-11-18T20:42:10,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/f76c1f30d57946489b7a5f8beff25288, entries=150, sequenceid=300, filesize=12.0 K 2024-11-18T20:42:10,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=140.89 KB/144270 for 9fe934dbc79ce3fabcdcf715bb36d192 in 206ms, sequenceid=300, compaction requested=true 2024-11-18T20:42:10,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:10,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:42:10,482 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:10,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:10,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:42:10,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:10,482 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:10,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:42:10,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:10,483 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:10,483 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:10,483 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/B is initiating minor compaction (all files) 2024-11-18T20:42:10,483 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/A is initiating minor compaction (all files) 2024-11-18T20:42:10,483 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/B in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:10,483 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/d2ce26dcad284f6faaaa3a05e0e1e9a8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/7d2c96a55dc544359c115650a28dd372, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/1f7d13ef2eee4c819bfe01fcd60998bc] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=36.6 K 2024-11-18T20:42:10,483 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/A in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:10,483 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5e85b61a0af44e0fa219c4c4f36e7686, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/290fca8c1a8e484496225917641cb806, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/64c7f396241a4585a884f253132d792d] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=39.0 K 2024-11-18T20:42:10,484 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting d2ce26dcad284f6faaaa3a05e0e1e9a8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731962528274 2024-11-18T20:42:10,484 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e85b61a0af44e0fa219c4c4f36e7686, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731962528274 2024-11-18T20:42:10,484 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d2c96a55dc544359c115650a28dd372, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731962528937 2024-11-18T20:42:10,484 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 290fca8c1a8e484496225917641cb806, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731962528937 2024-11-18T20:42:10,484 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f7d13ef2eee4c819bfe01fcd60998bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731962530270 2024-11-18T20:42:10,484 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64c7f396241a4585a884f253132d792d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731962530154 2024-11-18T20:42:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:10,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=147.60 KB heapSize=414.61 KB 2024-11-18T20:42:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:42:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:42:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:42:10,493 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#B#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:10,493 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#A#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:10,493 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/396bc5af8b084f7ebe29e778fe9fc618 is 50, key is test_row_0/A:col10/1731962530273/Put/seqid=0 2024-11-18T20:42:10,493 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/0c67e950812b4d2f8117e0775e26d8e2 is 50, key is test_row_0/B:col10/1731962530273/Put/seqid=0 2024-11-18T20:42:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T20:42:10,497 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:10,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:42:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-18T20:42:10,500 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:42:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T20:42:10,500 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:42:10,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:42:10,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962590498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:10,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:10,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962590499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:10,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/19073208f1ee4b4faf24e8ddac589761 is 50, key is test_row_0/A:col10/1731962530487/Put/seqid=0 2024-11-18T20:42:10,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:10,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962590501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:10,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:10,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742133_1309 (size=13051) 2024-11-18T20:42:10,522 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/0c67e950812b4d2f8117e0775e26d8e2 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/0c67e950812b4d2f8117e0775e26d8e2 2024-11-18T20:42:10,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742134_1310 (size=13051) 2024-11-18T20:42:10,529 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/B of 9fe934dbc79ce3fabcdcf715bb36d192 into 0c67e950812b4d2f8117e0775e26d8e2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:10,529 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:10,529 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/B, priority=13, startTime=1731962530482; duration=0sec 2024-11-18T20:42:10,529 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:10,529 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:B 2024-11-18T20:42:10,530 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-18T20:42:10,531 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T20:42:10,531 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/396bc5af8b084f7ebe29e778fe9fc618 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/396bc5af8b084f7ebe29e778fe9fc618 2024-11-18T20:42:10,532 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T20:42:10,532 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. because compaction request was cancelled 2024-11-18T20:42:10,532 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:C 2024-11-18T20:42:10,538 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/A of 9fe934dbc79ce3fabcdcf715bb36d192 into 396bc5af8b084f7ebe29e778fe9fc618(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:10,538 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:10,538 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/A, priority=13, startTime=1731962530482; duration=0sec 2024-11-18T20:42:10,538 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:10,538 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:A 2024-11-18T20:42:10,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742135_1311 (size=14741) 2024-11-18T20:42:10,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/19073208f1ee4b4faf24e8ddac589761 2024-11-18T20:42:10,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/b841382686fb4b01beaeb38ab1e978a8 is 50, key is test_row_0/B:col10/1731962530487/Put/seqid=0 2024-11-18T20:42:10,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742136_1312 (size=12301) 2024-11-18T20:42:10,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/b841382686fb4b01beaeb38ab1e978a8 2024-11-18T20:42:10,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/858227cd71394efe8aa7b2be19d78940 is 50, key is test_row_0/C:col10/1731962530487/Put/seqid=0 2024-11-18T20:42:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T20:42:10,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742137_1313 (size=12301) 2024-11-18T20:42:10,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/858227cd71394efe8aa7b2be19d78940 2024-11-18T20:42:10,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962590608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962590608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:10,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/19073208f1ee4b4faf24e8ddac589761 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/19073208f1ee4b4faf24e8ddac589761 2024-11-18T20:42:10,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962590618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/19073208f1ee4b4faf24e8ddac589761, entries=200, sequenceid=357, filesize=14.4 K 2024-11-18T20:42:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:10,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/b841382686fb4b01beaeb38ab1e978a8 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/b841382686fb4b01beaeb38ab1e978a8 2024-11-18T20:42:10,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/b841382686fb4b01beaeb38ab1e978a8, entries=150, sequenceid=357, filesize=12.0 K 2024-11-18T20:42:10,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/858227cd71394efe8aa7b2be19d78940 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/858227cd71394efe8aa7b2be19d78940 2024-11-18T20:42:10,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/858227cd71394efe8aa7b2be19d78940, entries=150, sequenceid=357, filesize=12.0 K 2024-11-18T20:42:10,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~414.56 KB/424512, currentSize=33.54 KB/34350 for 9fe934dbc79ce3fabcdcf715bb36d192 in 151ms, sequenceid=357, compaction requested=true 2024-11-18T20:42:10,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:10,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:42:10,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:10,639 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-18T20:42:10,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:42:10,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:10,639 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-18T20:42:10,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:42:10,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:10,640 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T20:42:10,640 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T20:42:10,640 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T20:42:10,640 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T20:42:10,640 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. because compaction request was cancelled 2024-11-18T20:42:10,640 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. because compaction request was cancelled 2024-11-18T20:42:10,640 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:A 2024-11-18T20:42:10,640 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:B 2024-11-18T20:42:10,640 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:10,641 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:10,641 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/C is initiating minor compaction (all files) 2024-11-18T20:42:10,641 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/C in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:10,641 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/d2fd99b4c3a04b4a9674f489826fecc6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/f76c1f30d57946489b7a5f8beff25288, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/858227cd71394efe8aa7b2be19d78940] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=36.7 K 2024-11-18T20:42:10,641 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2fd99b4c3a04b4a9674f489826fecc6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731962528937 2024-11-18T20:42:10,642 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting f76c1f30d57946489b7a5f8beff25288, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731962530270 2024-11-18T20:42:10,645 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 858227cd71394efe8aa7b2be19d78940, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731962530479 2024-11-18T20:42:10,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-18T20:42:10,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:10,656 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-18T20:42:10,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:10,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:10,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:10,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:10,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:10,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:10,667 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#C#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:10,668 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/e36f165857194f80b810cce51606af1e is 50, key is test_row_0/C:col10/1731962530487/Put/seqid=0 2024-11-18T20:42:10,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/e9db33d80ee84168b1053b0cfc614323 is 50, key is test_row_1/A:col10/1731962530498/Put/seqid=0 2024-11-18T20:42:10,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742138_1314 (size=13085) 2024-11-18T20:42:10,721 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/e36f165857194f80b810cce51606af1e as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/e36f165857194f80b810cce51606af1e 2024-11-18T20:42:10,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742139_1315 (size=9857) 2024-11-18T20:42:10,723 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/e9db33d80ee84168b1053b0cfc614323 2024-11-18T20:42:10,754 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/C of 9fe934dbc79ce3fabcdcf715bb36d192 into e36f165857194f80b810cce51606af1e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:10,754 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:10,754 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/C, priority=13, startTime=1731962530639; duration=0sec 2024-11-18T20:42:10,754 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:10,754 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:C 2024-11-18T20:42:10,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/cf3fb5430a654daeac8381f273832fd3 is 50, key is test_row_1/B:col10/1731962530498/Put/seqid=0 2024-11-18T20:42:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T20:42:10,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742140_1316 (size=9857) 2024-11-18T20:42:10,827 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/cf3fb5430a654daeac8381f273832fd3 2024-11-18T20:42:10,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:10,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/c3400a0973bb4688b622e9d092683cf2 is 50, key is test_row_1/C:col10/1731962530498/Put/seqid=0 2024-11-18T20:42:10,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742141_1317 (size=9857) 2024-11-18T20:42:10,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962530850, max timestamp=1731962530882Num uniques 859; 2024-11-18T20:42:10,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:42:10,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962530850, max timestamp=1731962530882Num uniques 859; 2024-11-18T20:42:10,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:42:10,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962530850, max timestamp=1731962530882Num uniques 859; 2024-11-18T20:42:10,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:42:10,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962590933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:10,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731962530918, max timestamp=1731962530934Num uniques 1068; 2024-11-18T20:42:10,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-18T20:42:10,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731962530918, max timestamp=1731962530934Num uniques 1068; 2024-11-18T20:42:10,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-18T20:42:10,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731962530918, max timestamp=1731962530934Num uniques 1068; 2024-11-18T20:42:10,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-18T20:42:11,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962591007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:11,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731962530997, max timestamp=1731962531014Num uniques 1077; 2024-11-18T20:42:11,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731962530997, max timestamp=1731962531014Num uniques 1077; 2024-11-18T20:42:11,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-18T20:42:11,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-18T20:42:11,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731962530997, max timestamp=1731962531014Num uniques 1077; 2024-11-18T20:42:11,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-18T20:42:11,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731962531087, max timestamp=1731962531116Num uniques 1036; 2024-11-18T20:42:11,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-18T20:42:11,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731962531087, max timestamp=1731962531116Num uniques 1036; 2024-11-18T20:42:11,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-18T20:42:11,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731962531087, max timestamp=1731962531116Num uniques 1036; 2024-11-18T20:42:11,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-18T20:42:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T20:42:11,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11210, totalHeapSize=25720, min timestamp=1731962531153, max timestamp=1731962531215Num uniques 1045; 2024-11-18T20:42:11,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47850, new segment data size=11210, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-18T20:42:11,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731962531153, max timestamp=1731962531215Num uniques 1045; 2024-11-18T20:42:11,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=6870, suffix heap size=136648,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=150 2024-11-18T20:42:11,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11210, totalHeapSize=25720, min timestamp=1731962531153, max timestamp=1731962531215Num uniques 1045; 2024-11-18T20:42:11,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47850, new segment data size=11210, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-18T20:42:11,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962591261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:11,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:11,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962591262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:11,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962591264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:11,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731962531247, max timestamp=1731962531262Num uniques 1009; 2024-11-18T20:42:11,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=7050, suffix heap size=132904,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=154 2024-11-18T20:42:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:11,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731962531247, max timestamp=1731962531262Num uniques 1104; 2024-11-18T20:42:11,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-18T20:42:11,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731962531247, max timestamp=1731962531262Num uniques 1104; 2024-11-18T20:42:11,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-18T20:42:11,276 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/c3400a0973bb4688b622e9d092683cf2 2024-11-18T20:42:11,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/e9db33d80ee84168b1053b0cfc614323 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/e9db33d80ee84168b1053b0cfc614323 2024-11-18T20:42:11,292 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/e9db33d80ee84168b1053b0cfc614323, entries=100, sequenceid=367, filesize=9.6 K 2024-11-18T20:42:11,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/cf3fb5430a654daeac8381f273832fd3 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/cf3fb5430a654daeac8381f273832fd3 2024-11-18T20:42:11,305 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/cf3fb5430a654daeac8381f273832fd3, entries=100, sequenceid=367, filesize=9.6 K 2024-11-18T20:42:11,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/c3400a0973bb4688b622e9d092683cf2 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/c3400a0973bb4688b622e9d092683cf2 2024-11-18T20:42:11,312 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/c3400a0973bb4688b622e9d092683cf2, entries=100, sequenceid=367, filesize=9.6 K 2024-11-18T20:42:11,314 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=26.84 KB/27480 for 9fe934dbc79ce3fabcdcf715bb36d192 in 657ms, sequenceid=367, compaction requested=true 2024-11-18T20:42:11,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:11,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:11,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-18T20:42:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-18T20:42:11,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-18T20:42:11,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 816 msec 2024-11-18T20:42:11,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 824 msec 2024-11-18T20:42:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:11,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=53.67 KB heapSize=146.39 KB 2024-11-18T20:42:11,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:11,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:42:11,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:11,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:42:11,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:11,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:42:11,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/79f0c93745ff4e08a95d97d35ae39033 is 50, key is test_row_0/A:col10/1731962531262/Put/seqid=0 2024-11-18T20:42:11,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962591433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:11,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731962531410, max timestamp=1731962531434Num uniques 859; 2024-11-18T20:42:11,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T20:42:11,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731962531410, max timestamp=1731962531434Num uniques 859; 2024-11-18T20:42:11,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T20:42:11,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731962531410, max timestamp=1731962531434Num uniques 859; 2024-11-18T20:42:11,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T20:42:11,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742142_1318 (size=19621) 2024-11-18T20:42:11,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/79f0c93745ff4e08a95d97d35ae39033 2024-11-18T20:42:11,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/f9dfd326902d48af95fa8e58b510441c is 50, key is test_row_0/B:col10/1731962531262/Put/seqid=0 2024-11-18T20:42:11,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742143_1319 (size=12301) 2024-11-18T20:42:11,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962591496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:11,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962591499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:11,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962591547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:11,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:11,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962591607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:11,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962591607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:11,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T20:42:11,637 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:11,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:42:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-18T20:42:11,640 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:42:11,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T20:42:11,640 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:42:11,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:42:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T20:42:11,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:11,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962591757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:11,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:11,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T20:42:11,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:11,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:11,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:11,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:11,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:11,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962591817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:11,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962591817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:11,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/f9dfd326902d48af95fa8e58b510441c 2024-11-18T20:42:11,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/edcea59a73124d87aeb1865d7c1fbb06 is 50, key is test_row_0/C:col10/1731962531262/Put/seqid=0 2024-11-18T20:42:11,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742144_1320 (size=12301) 2024-11-18T20:42:11,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T20:42:11,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:11,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:11,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:11,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:11,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T20:42:12,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:12,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962592068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:12,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:12,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:12,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:12,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T20:42:12,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:12,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:12,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:12,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:12,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:12,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962592127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:12,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962592127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:12,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:12,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:12,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:12,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T20:42:12,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:12,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:12,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:12,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:12,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T20:42:12,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/edcea59a73124d87aeb1865d7c1fbb06 2024-11-18T20:42:12,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/79f0c93745ff4e08a95d97d35ae39033 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/79f0c93745ff4e08a95d97d35ae39033 2024-11-18T20:42:12,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/79f0c93745ff4e08a95d97d35ae39033, entries=300, sequenceid=480, filesize=19.2 K 2024-11-18T20:42:12,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/f9dfd326902d48af95fa8e58b510441c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/f9dfd326902d48af95fa8e58b510441c 2024-11-18T20:42:12,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/f9dfd326902d48af95fa8e58b510441c, entries=150, sequenceid=480, filesize=12.0 K 2024-11-18T20:42:12,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/edcea59a73124d87aeb1865d7c1fbb06 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/edcea59a73124d87aeb1865d7c1fbb06 2024-11-18T20:42:12,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/edcea59a73124d87aeb1865d7c1fbb06, entries=150, sequenceid=480, filesize=12.0 K 2024-11-18T20:42:12,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~166.27 KB/170256, currentSize=127.47 KB/130530 for 9fe934dbc79ce3fabcdcf715bb36d192 in 918ms, sequenceid=480, compaction requested=true 2024-11-18T20:42:12,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:12,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:42:12,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:12,301 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:42:12,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:42:12,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:12,301 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:42:12,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:42:12,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:12,303 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57270 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:42:12,303 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:42:12,303 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/A is initiating minor compaction (all files) 2024-11-18T20:42:12,303 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/B is initiating minor compaction (all files) 2024-11-18T20:42:12,303 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/B in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:12,303 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/A in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:12,303 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/0c67e950812b4d2f8117e0775e26d8e2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/b841382686fb4b01beaeb38ab1e978a8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/cf3fb5430a654daeac8381f273832fd3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/f9dfd326902d48af95fa8e58b510441c] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=46.4 K 2024-11-18T20:42:12,303 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/396bc5af8b084f7ebe29e778fe9fc618, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/19073208f1ee4b4faf24e8ddac589761, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/e9db33d80ee84168b1053b0cfc614323, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/79f0c93745ff4e08a95d97d35ae39033] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=55.9 K 2024-11-18T20:42:12,303 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c67e950812b4d2f8117e0775e26d8e2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731962530270 2024-11-18T20:42:12,304 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 396bc5af8b084f7ebe29e778fe9fc618, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731962530270 2024-11-18T20:42:12,304 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting b841382686fb4b01beaeb38ab1e978a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731962530479 2024-11-18T20:42:12,304 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19073208f1ee4b4faf24e8ddac589761, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731962530469 2024-11-18T20:42:12,304 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting cf3fb5430a654daeac8381f273832fd3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731962530495 2024-11-18T20:42:12,305 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9db33d80ee84168b1053b0cfc614323, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731962530495 2024-11-18T20:42:12,305 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting f9dfd326902d48af95fa8e58b510441c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731962531262 2024-11-18T20:42:12,305 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79f0c93745ff4e08a95d97d35ae39033, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731962531257 2024-11-18T20:42:12,324 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#A#compaction#275 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:12,324 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/5eca31ac83f4484ea526dd8ba8ccbfe7 is 50, key is test_row_0/A:col10/1731962531262/Put/seqid=0 2024-11-18T20:42:12,328 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#B#compaction#276 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:12,328 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/6d55f52ec1aa49b69fdf355c8357653c is 50, key is test_row_0/B:col10/1731962531262/Put/seqid=0 2024-11-18T20:42:12,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742145_1321 (size=13187) 2024-11-18T20:42:12,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742146_1322 (size=13187) 2024-11-18T20:42:12,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T20:42:12,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:12,406 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-18T20:42:12,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:12,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:42:12,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:12,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:42:12,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:12,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:42:12,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/c8c4761498a640419f6936c68f0ee276 is 50, key is test_row_0/A:col10/1731962531485/Put/seqid=0 2024-11-18T20:42:12,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742147_1323 (size=12301) 2024-11-18T20:42:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:12,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:12,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962592633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:12,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:12,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:12,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:12,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962592638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:12,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962592638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:12,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962592747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:12,760 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/6d55f52ec1aa49b69fdf355c8357653c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/6d55f52ec1aa49b69fdf355c8357653c 2024-11-18T20:42:12,765 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/B of 9fe934dbc79ce3fabcdcf715bb36d192 into 6d55f52ec1aa49b69fdf355c8357653c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:12,765 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:12,765 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/B, priority=12, startTime=1731962532301; duration=0sec 2024-11-18T20:42:12,765 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:12,765 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:B 2024-11-18T20:42:12,765 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:12,766 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:12,766 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/C is initiating minor compaction (all files) 2024-11-18T20:42:12,766 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/C in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:12,766 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/e36f165857194f80b810cce51606af1e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/c3400a0973bb4688b622e9d092683cf2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/edcea59a73124d87aeb1865d7c1fbb06] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=34.4 K 2024-11-18T20:42:12,767 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting e36f165857194f80b810cce51606af1e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731962530479 2024-11-18T20:42:12,767 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting c3400a0973bb4688b622e9d092683cf2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731962530495 2024-11-18T20:42:12,767 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting edcea59a73124d87aeb1865d7c1fbb06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731962531262 2024-11-18T20:42:12,775 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/5eca31ac83f4484ea526dd8ba8ccbfe7 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5eca31ac83f4484ea526dd8ba8ccbfe7 2024-11-18T20:42:12,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T20:42:12,780 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#C#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:12,781 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/43b8b9a1265d43fe9658a74616c8e15e is 50, key is test_row_0/C:col10/1731962531262/Put/seqid=0 2024-11-18T20:42:12,784 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/A of 9fe934dbc79ce3fabcdcf715bb36d192 into 5eca31ac83f4484ea526dd8ba8ccbfe7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:12,784 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:12,784 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/A, priority=12, startTime=1731962532301; duration=0sec 2024-11-18T20:42:12,784 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:12,784 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:A 2024-11-18T20:42:12,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742148_1324 (size=13187) 2024-11-18T20:42:12,792 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/43b8b9a1265d43fe9658a74616c8e15e as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/43b8b9a1265d43fe9658a74616c8e15e 2024-11-18T20:42:12,809 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/C of 9fe934dbc79ce3fabcdcf715bb36d192 into 43b8b9a1265d43fe9658a74616c8e15e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:12,809 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:12,809 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/C, priority=13, startTime=1731962532301; duration=0sec 2024-11-18T20:42:12,809 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:12,809 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:C 2024-11-18T20:42:12,815 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/c8c4761498a640419f6936c68f0ee276 2024-11-18T20:42:12,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/ff84483c58da4412a2b59b72164ed135 is 50, key is test_row_0/B:col10/1731962531485/Put/seqid=0 2024-11-18T20:42:12,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742149_1325 (size=12301) 2024-11-18T20:42:12,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962592957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:13,228 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/ff84483c58da4412a2b59b72164ed135 2024-11-18T20:42:13,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/01af5ee4199842fb9d43864d8838721d is 50, key is test_row_0/C:col10/1731962531485/Put/seqid=0 2024-11-18T20:42:13,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742150_1326 (size=12301) 2024-11-18T20:42:13,239 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/01af5ee4199842fb9d43864d8838721d 2024-11-18T20:42:13,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/c8c4761498a640419f6936c68f0ee276 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/c8c4761498a640419f6936c68f0ee276 2024-11-18T20:42:13,249 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/c8c4761498a640419f6936c68f0ee276, entries=150, sequenceid=514, filesize=12.0 K 2024-11-18T20:42:13,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/ff84483c58da4412a2b59b72164ed135 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/ff84483c58da4412a2b59b72164ed135 2024-11-18T20:42:13,258 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/ff84483c58da4412a2b59b72164ed135, entries=150, sequenceid=514, filesize=12.0 K 2024-11-18T20:42:13,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/01af5ee4199842fb9d43864d8838721d as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/01af5ee4199842fb9d43864d8838721d 2024-11-18T20:42:13,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962593267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:13,279 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/01af5ee4199842fb9d43864d8838721d, entries=150, sequenceid=514, filesize=12.0 K 2024-11-18T20:42:13,280 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 9fe934dbc79ce3fabcdcf715bb36d192 in 874ms, sequenceid=514, compaction requested=false 2024-11-18T20:42:13,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:13,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:13,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-18T20:42:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-18T20:42:13,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-18T20:42:13,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6430 sec 2024-11-18T20:42:13,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.6530 sec 2024-11-18T20:42:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:13,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T20:42:13,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:13,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:13,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:13,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:13,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:13,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:13,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/5b21d8dd399d43cb9fe6d0294647ae44 is 50, key is test_row_0/A:col10/1731962533659/Put/seqid=0 2024-11-18T20:42:13,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742151_1327 (size=14741) 2024-11-18T20:42:13,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/5b21d8dd399d43cb9fe6d0294647ae44 2024-11-18T20:42:13,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/581fdb6985304b66bdab10a22a9289da is 50, key is test_row_0/B:col10/1731962533659/Put/seqid=0 2024-11-18T20:42:13,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742152_1328 (size=12301) 2024-11-18T20:42:13,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/581fdb6985304b66bdab10a22a9289da 2024-11-18T20:42:13,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/706d5d17c3e249eca8a0d34bb90e176b is 50, key is test_row_0/C:col10/1731962533659/Put/seqid=0 2024-11-18T20:42:13,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962593758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:13,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742153_1329 (size=12301) 2024-11-18T20:42:13,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/706d5d17c3e249eca8a0d34bb90e176b 2024-11-18T20:42:13,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:13,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962593761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:13,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/5b21d8dd399d43cb9fe6d0294647ae44 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5b21d8dd399d43cb9fe6d0294647ae44 2024-11-18T20:42:13,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5b21d8dd399d43cb9fe6d0294647ae44, entries=200, sequenceid=530, filesize=14.4 K 2024-11-18T20:42:13,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/581fdb6985304b66bdab10a22a9289da as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/581fdb6985304b66bdab10a22a9289da 2024-11-18T20:42:13,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:13,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962593778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:13,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:13,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:13,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:13,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/581fdb6985304b66bdab10a22a9289da, entries=150, sequenceid=530, filesize=12.0 K 2024-11-18T20:42:13,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/706d5d17c3e249eca8a0d34bb90e176b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/706d5d17c3e249eca8a0d34bb90e176b 2024-11-18T20:42:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T20:42:13,789 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:13,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:42:13,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-18T20:42:13,795 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:42:13,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T20:42:13,797 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:42:13,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:42:13,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/706d5d17c3e249eca8a0d34bb90e176b, entries=150, sequenceid=530, filesize=12.0 K 2024-11-18T20:42:13,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9fe934dbc79ce3fabcdcf715bb36d192 in 144ms, sequenceid=530, compaction requested=true 2024-11-18T20:42:13,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:13,806 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:13,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:42:13,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:13,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:42:13,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:13,806 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:13,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:42:13,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:13,807 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:13,807 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/A is initiating minor compaction (all files) 2024-11-18T20:42:13,807 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/A in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:13,807 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:13,807 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/B is initiating minor compaction (all files) 2024-11-18T20:42:13,807 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/B in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:13,807 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5eca31ac83f4484ea526dd8ba8ccbfe7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/c8c4761498a640419f6936c68f0ee276, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5b21d8dd399d43cb9fe6d0294647ae44] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=39.3 K 2024-11-18T20:42:13,807 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/6d55f52ec1aa49b69fdf355c8357653c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/ff84483c58da4412a2b59b72164ed135, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/581fdb6985304b66bdab10a22a9289da] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=36.9 K 2024-11-18T20:42:13,808 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5eca31ac83f4484ea526dd8ba8ccbfe7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731962531262 2024-11-18T20:42:13,808 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d55f52ec1aa49b69fdf355c8357653c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731962531262 2024-11-18T20:42:13,808 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting ff84483c58da4412a2b59b72164ed135, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731962531485 2024-11-18T20:42:13,808 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8c4761498a640419f6936c68f0ee276, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731962531485 2024-11-18T20:42:13,808 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 581fdb6985304b66bdab10a22a9289da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1731962532588 2024-11-18T20:42:13,809 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b21d8dd399d43cb9fe6d0294647ae44, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1731962532588 2024-11-18T20:42:13,823 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#B#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:13,824 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/035e26c7277e44febb87070f029f6fd9 is 50, key is test_row_0/B:col10/1731962533659/Put/seqid=0 2024-11-18T20:42:13,839 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#A#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:13,840 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/739b8bc95d3d4636914b8c5c78a8c236 is 50, key is test_row_0/A:col10/1731962533659/Put/seqid=0 2024-11-18T20:42:13,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742154_1330 (size=13289) 2024-11-18T20:42:13,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742155_1331 (size=13289) 2024-11-18T20:42:13,857 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/739b8bc95d3d4636914b8c5c78a8c236 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/739b8bc95d3d4636914b8c5c78a8c236 2024-11-18T20:42:13,857 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/035e26c7277e44febb87070f029f6fd9 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/035e26c7277e44febb87070f029f6fd9 2024-11-18T20:42:13,865 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/A of 9fe934dbc79ce3fabcdcf715bb36d192 into 739b8bc95d3d4636914b8c5c78a8c236(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:13,865 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:13,865 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/A, priority=13, startTime=1731962533805; duration=0sec 2024-11-18T20:42:13,865 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:13,865 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:A 2024-11-18T20:42:13,865 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:13,865 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/B of 9fe934dbc79ce3fabcdcf715bb36d192 into 035e26c7277e44febb87070f029f6fd9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:13,865 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:13,866 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/B, priority=13, startTime=1731962533806; duration=0sec 2024-11-18T20:42:13,866 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:13,866 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:B 2024-11-18T20:42:13,868 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:13,868 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/C is initiating minor compaction (all files) 2024-11-18T20:42:13,868 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/C in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:13,869 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/43b8b9a1265d43fe9658a74616c8e15e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/01af5ee4199842fb9d43864d8838721d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/706d5d17c3e249eca8a0d34bb90e176b] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=36.9 K 2024-11-18T20:42:13,869 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43b8b9a1265d43fe9658a74616c8e15e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731962531262 2024-11-18T20:42:13,869 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01af5ee4199842fb9d43864d8838721d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731962531485 2024-11-18T20:42:13,870 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 706d5d17c3e249eca8a0d34bb90e176b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1731962532588 2024-11-18T20:42:13,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T20:42:13,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:13,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:13,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:13,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:13,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:13,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:13,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/3bf289e95d7b4409a3f1af4409fb6a48 is 50, key is test_row_0/A:col10/1731962533872/Put/seqid=0 2024-11-18T20:42:13,888 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#C#compaction#287 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:13,888 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/0b3a5b59ca1b4aa891eda176fed4705a is 50, key is test_row_0/C:col10/1731962533659/Put/seqid=0 2024-11-18T20:42:13,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T20:42:13,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742156_1332 (size=12301) 2024-11-18T20:42:13,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962593924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:13,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:13,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/3bf289e95d7b4409a3f1af4409fb6a48 2024-11-18T20:42:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962593924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:13,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:13,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:13,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:13,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:13,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742157_1333 (size=13289) 2024-11-18T20:42:13,941 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/0b3a5b59ca1b4aa891eda176fed4705a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/0b3a5b59ca1b4aa891eda176fed4705a 2024-11-18T20:42:13,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/394cad9551d2467fa46ac5e1ddd36c91 is 50, key is test_row_0/B:col10/1731962533872/Put/seqid=0 2024-11-18T20:42:13,949 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/C of 9fe934dbc79ce3fabcdcf715bb36d192 into 0b3a5b59ca1b4aa891eda176fed4705a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:13,949 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:13,949 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/C, priority=13, startTime=1731962533806; duration=0sec 2024-11-18T20:42:13,949 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:13,949 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:C 2024-11-18T20:42:13,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-18T20:42:13,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:13,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:13,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:13,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:13,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:13,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:13,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742158_1334 (size=12301) 2024-11-18T20:42:13,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/394cad9551d2467fa46ac5e1ddd36c91 2024-11-18T20:42:13,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32874 deadline: 1731962593962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:13,965 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in 9fe934dbc79ce3fabcdcf715bb36d192 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18285 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:42:13,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:13,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:13,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:13,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:13,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32890 deadline: 1731962593968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:13,973 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in 9fe934dbc79ce3fabcdcf715bb36d192 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18295 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:42:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:13,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/41c6feb8a66e43078fd2cee0e2ba165c is 50, key is test_row_0/C:col10/1731962533872/Put/seqid=0 2024-11-18T20:42:14,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742159_1335 (size=12301) 2024-11-18T20:42:14,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/41c6feb8a66e43078fd2cee0e2ba165c 2024-11-18T20:42:14,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/3bf289e95d7b4409a3f1af4409fb6a48 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/3bf289e95d7b4409a3f1af4409fb6a48 2024-11-18T20:42:14,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962594037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:14,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:14,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/3bf289e95d7b4409a3f1af4409fb6a48, entries=150, sequenceid=552, filesize=12.0 K 2024-11-18T20:42:14,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/394cad9551d2467fa46ac5e1ddd36c91 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/394cad9551d2467fa46ac5e1ddd36c91 2024-11-18T20:42:14,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962594041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:14,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/394cad9551d2467fa46ac5e1ddd36c91, entries=150, sequenceid=552, filesize=12.0 K 2024-11-18T20:42:14,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/41c6feb8a66e43078fd2cee0e2ba165c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/41c6feb8a66e43078fd2cee0e2ba165c 2024-11-18T20:42:14,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/41c6feb8a66e43078fd2cee0e2ba165c, entries=150, sequenceid=552, filesize=12.0 K 2024-11-18T20:42:14,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 9fe934dbc79ce3fabcdcf715bb36d192 in 186ms, sequenceid=552, compaction requested=false 2024-11-18T20:42:14,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:14,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-18T20:42:14,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:14,106 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T20:42:14,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:14,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:14,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:14,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:14,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:14,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:14,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/7152def1207e4a95aba07afde0057d5a is 50, key is test_row_0/A:col10/1731962533901/Put/seqid=0 2024-11-18T20:42:14,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742160_1336 (size=12301) 2024-11-18T20:42:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T20:42:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:14,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. as already flushing 2024-11-18T20:42:14,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962594303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:14,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962594304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:14,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962594417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:14,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:14,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962594417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:14,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:14,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T20:42:14,515 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=566 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/7152def1207e4a95aba07afde0057d5a 2024-11-18T20:42:14,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/fe37f852611f4f0fafddffdf95450f6d is 50, key is test_row_0/B:col10/1731962533901/Put/seqid=0 2024-11-18T20:42:14,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742161_1337 (size=12301) 2024-11-18T20:42:14,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962594628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:14,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962594628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:14,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32948 deadline: 1731962594787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:14,791 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in 9fe934dbc79ce3fabcdcf715bb36d192 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:42:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T20:42:14,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962594937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:14,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962594937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:14,950 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=566 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/fe37f852611f4f0fafddffdf95450f6d 2024-11-18T20:42:14,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/fbf1f935c7054558879c5703a97622a1 is 50, key is test_row_0/C:col10/1731962533901/Put/seqid=0 2024-11-18T20:42:14,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742162_1338 (size=12301) 2024-11-18T20:42:15,384 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=566 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/fbf1f935c7054558879c5703a97622a1 2024-11-18T20:42:15,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/7152def1207e4a95aba07afde0057d5a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/7152def1207e4a95aba07afde0057d5a 2024-11-18T20:42:15,395 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/7152def1207e4a95aba07afde0057d5a, entries=150, sequenceid=566, filesize=12.0 K 2024-11-18T20:42:15,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/fe37f852611f4f0fafddffdf95450f6d as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/fe37f852611f4f0fafddffdf95450f6d 2024-11-18T20:42:15,401 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/fe37f852611f4f0fafddffdf95450f6d, entries=150, sequenceid=566, filesize=12.0 K 2024-11-18T20:42:15,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/fbf1f935c7054558879c5703a97622a1 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/fbf1f935c7054558879c5703a97622a1 2024-11-18T20:42:15,406 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/fbf1f935c7054558879c5703a97622a1, entries=150, sequenceid=566, filesize=12.0 K 2024-11-18T20:42:15,406 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9fe934dbc79ce3fabcdcf715bb36d192 in 1300ms, sequenceid=566, compaction requested=true 2024-11-18T20:42:15,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:15,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:15,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-18T20:42:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-18T20:42:15,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-18T20:42:15,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6110 sec 2024-11-18T20:42:15,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.6270 sec 2024-11-18T20:42:15,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T20:42:15,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:15,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:15,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:15,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:15,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:15,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:15,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/56031ec9726441bd8e5f7a20f99b9902 is 50, key is test_row_0/A:col10/1731962535450/Put/seqid=0 2024-11-18T20:42:15,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742163_1339 (size=12301) 2024-11-18T20:42:15,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/56031ec9726441bd8e5f7a20f99b9902 2024-11-18T20:42:15,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962595518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:15,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962595523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:15,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:15,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:15,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:15,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:15,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:15,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:15,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/891ef3f0c32c424bbf9a22d4c6ad6a6d is 50, key is test_row_0/B:col10/1731962535450/Put/seqid=0 2024-11-18T20:42:15,597 INFO [Thread-1143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1143. 2024-11-18T20:42:15,597 DEBUG [Thread-1143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:15,597 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:15,597 INFO [Thread-1147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1147. 2024-11-18T20:42:15,597 DEBUG [Thread-1147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:15,597 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:15,597 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:15,597 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:15,598 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:15,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:15,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742164_1340 (size=12301) 2024-11-18T20:42:15,600 INFO [Thread-1145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1145. 2024-11-18T20:42:15,600 DEBUG [Thread-1145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:15,600 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:15,600 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:15,600 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:15,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/891ef3f0c32c424bbf9a22d4c6ad6a6d 2024-11-18T20:42:15,602 INFO [Thread-1141 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1141. 2024-11-18T20:42:15,602 DEBUG [Thread-1141 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:15,602 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:15,602 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:15,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:15,608 INFO [Thread-1139 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1139. 2024-11-18T20:42:15,608 DEBUG [Thread-1139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:15,608 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:15,608 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:15,608 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:15,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/70cc9453a325406bbf70d812defd66a0 is 50, key is test_row_0/C:col10/1731962535450/Put/seqid=0 2024-11-18T20:42:15,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742165_1341 (size=12301) 2024-11-18T20:42:15,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/70cc9453a325406bbf70d812defd66a0 2024-11-18T20:42:15,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/56031ec9726441bd8e5f7a20f99b9902 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/56031ec9726441bd8e5f7a20f99b9902 2024-11-18T20:42:15,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32896 deadline: 1731962595638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:15,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32932 deadline: 1731962595639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:42:15,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/56031ec9726441bd8e5f7a20f99b9902, entries=150, sequenceid=586, filesize=12.0 K 2024-11-18T20:42:15,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/891ef3f0c32c424bbf9a22d4c6ad6a6d as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/891ef3f0c32c424bbf9a22d4c6ad6a6d 2024-11-18T20:42:15,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/891ef3f0c32c424bbf9a22d4c6ad6a6d, entries=150, sequenceid=586, filesize=12.0 K 2024-11-18T20:42:15,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/70cc9453a325406bbf70d812defd66a0 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/70cc9453a325406bbf70d812defd66a0 2024-11-18T20:42:15,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/70cc9453a325406bbf70d812defd66a0, entries=150, sequenceid=586, filesize=12.0 K 2024-11-18T20:42:15,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 9fe934dbc79ce3fabcdcf715bb36d192 in 214ms, sequenceid=586, compaction requested=true 2024-11-18T20:42:15,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:15,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:42:15,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:15,667 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:42:15,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:42:15,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:15,667 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:42:15,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fe934dbc79ce3fabcdcf715bb36d192:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:42:15,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:15,669 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:42:15,669 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:42:15,669 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/B is initiating minor compaction (all files) 2024-11-18T20:42:15,669 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/A is initiating minor compaction (all files) 2024-11-18T20:42:15,669 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/B in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:15,669 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/A in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:15,669 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/035e26c7277e44febb87070f029f6fd9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/394cad9551d2467fa46ac5e1ddd36c91, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/fe37f852611f4f0fafddffdf95450f6d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/891ef3f0c32c424bbf9a22d4c6ad6a6d] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=49.0 K 2024-11-18T20:42:15,669 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/739b8bc95d3d4636914b8c5c78a8c236, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/3bf289e95d7b4409a3f1af4409fb6a48, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/7152def1207e4a95aba07afde0057d5a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/56031ec9726441bd8e5f7a20f99b9902] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=49.0 K 2024-11-18T20:42:15,670 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 035e26c7277e44febb87070f029f6fd9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1731962532588 2024-11-18T20:42:15,670 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 739b8bc95d3d4636914b8c5c78a8c236, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1731962532588 2024-11-18T20:42:15,670 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 394cad9551d2467fa46ac5e1ddd36c91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1731962533749 2024-11-18T20:42:15,670 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bf289e95d7b4409a3f1af4409fb6a48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1731962533749 2024-11-18T20:42:15,670 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting fe37f852611f4f0fafddffdf95450f6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=566, earliestPutTs=1731962533900 2024-11-18T20:42:15,670 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7152def1207e4a95aba07afde0057d5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=566, earliestPutTs=1731962533900 2024-11-18T20:42:15,671 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 891ef3f0c32c424bbf9a22d4c6ad6a6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1731962534293 2024-11-18T20:42:15,671 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56031ec9726441bd8e5f7a20f99b9902, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1731962534293 2024-11-18T20:42:15,682 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#B#compaction#296 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:15,683 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/e76cfea2e1314fa0ac94955b75383083 is 50, key is test_row_0/B:col10/1731962535450/Put/seqid=0 2024-11-18T20:42:15,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742166_1342 (size=13425) 2024-11-18T20:42:15,688 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#A#compaction#297 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:15,688 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/1a1d1813ad314c29923a1b599195a6bb is 50, key is test_row_0/A:col10/1731962535450/Put/seqid=0 2024-11-18T20:42:15,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742167_1343 (size=13425) 2024-11-18T20:42:15,700 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/1a1d1813ad314c29923a1b599195a6bb as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/1a1d1813ad314c29923a1b599195a6bb 2024-11-18T20:42:15,706 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/A of 9fe934dbc79ce3fabcdcf715bb36d192 into 1a1d1813ad314c29923a1b599195a6bb(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:15,706 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:15,706 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/A, priority=12, startTime=1731962535667; duration=0sec 2024-11-18T20:42:15,706 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:15,706 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:A 2024-11-18T20:42:15,706 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:42:15,707 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:42:15,707 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 9fe934dbc79ce3fabcdcf715bb36d192/C is initiating minor compaction (all files) 2024-11-18T20:42:15,707 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fe934dbc79ce3fabcdcf715bb36d192/C in TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:15,708 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/0b3a5b59ca1b4aa891eda176fed4705a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/41c6feb8a66e43078fd2cee0e2ba165c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/fbf1f935c7054558879c5703a97622a1, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/70cc9453a325406bbf70d812defd66a0] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp, totalSize=49.0 K 2024-11-18T20:42:15,708 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b3a5b59ca1b4aa891eda176fed4705a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1731962532588 2024-11-18T20:42:15,709 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41c6feb8a66e43078fd2cee0e2ba165c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1731962533749 2024-11-18T20:42:15,709 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbf1f935c7054558879c5703a97622a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=566, earliestPutTs=1731962533900 2024-11-18T20:42:15,709 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70cc9453a325406bbf70d812defd66a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1731962534293 2024-11-18T20:42:15,718 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fe934dbc79ce3fabcdcf715bb36d192#C#compaction#298 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:15,718 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/5f98c0af3fae4767a252b16cb59465c1 is 50, key is test_row_0/C:col10/1731962535450/Put/seqid=0 2024-11-18T20:42:15,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742168_1344 (size=13425) 2024-11-18T20:42:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:15,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T20:42:15,849 INFO [Thread-1134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1134. 2024-11-18T20:42:15,849 DEBUG [Thread-1134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:15,849 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:15,849 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:15,850 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:15,850 INFO [Thread-1132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1132. 2024-11-18T20:42:15,850 DEBUG [Thread-1132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:15,850 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:15,850 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:15,850 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:15,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/01121a6318804143994c313788cbfe59 is 50, key is test_row_0/A:col10/1731962535848/Put/seqid=0 2024-11-18T20:42:15,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742169_1345 (size=12301) 2024-11-18T20:42:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T20:42:15,947 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:16,092 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/e76cfea2e1314fa0ac94955b75383083 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/e76cfea2e1314fa0ac94955b75383083 2024-11-18T20:42:16,096 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/B of 9fe934dbc79ce3fabcdcf715bb36d192 into e76cfea2e1314fa0ac94955b75383083(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:16,096 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:16,096 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/B, priority=12, startTime=1731962535667; duration=0sec 2024-11-18T20:42:16,096 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:16,096 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:B 2024-11-18T20:42:16,129 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/5f98c0af3fae4767a252b16cb59465c1 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/5f98c0af3fae4767a252b16cb59465c1 2024-11-18T20:42:16,136 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fe934dbc79ce3fabcdcf715bb36d192/C of 9fe934dbc79ce3fabcdcf715bb36d192 into 5f98c0af3fae4767a252b16cb59465c1(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:16,136 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:16,136 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., storeName=9fe934dbc79ce3fabcdcf715bb36d192/C, priority=12, startTime=1731962535668; duration=0sec 2024-11-18T20:42:16,136 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:16,136 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fe934dbc79ce3fabcdcf715bb36d192:C 2024-11-18T20:42:16,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=602 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/01121a6318804143994c313788cbfe59 2024-11-18T20:42:16,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/973a9c712f6c4dc1af1f27bc56c2649b is 50, key is test_row_0/B:col10/1731962535848/Put/seqid=0 2024-11-18T20:42:16,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742170_1346 (size=12301) 2024-11-18T20:42:16,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=602 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/973a9c712f6c4dc1af1f27bc56c2649b 2024-11-18T20:42:16,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/d8df14e6efe44b178a71f0a259f78b29 is 50, key is test_row_0/C:col10/1731962535848/Put/seqid=0 2024-11-18T20:42:16,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742171_1347 (size=12301) 2024-11-18T20:42:16,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=602 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/d8df14e6efe44b178a71f0a259f78b29 2024-11-18T20:42:16,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/01121a6318804143994c313788cbfe59 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/01121a6318804143994c313788cbfe59 2024-11-18T20:42:16,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/01121a6318804143994c313788cbfe59, entries=150, sequenceid=602, filesize=12.0 K 2024-11-18T20:42:16,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/973a9c712f6c4dc1af1f27bc56c2649b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/973a9c712f6c4dc1af1f27bc56c2649b 2024-11-18T20:42:16,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/973a9c712f6c4dc1af1f27bc56c2649b, entries=150, sequenceid=602, filesize=12.0 K 2024-11-18T20:42:16,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/d8df14e6efe44b178a71f0a259f78b29 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/d8df14e6efe44b178a71f0a259f78b29 2024-11-18T20:42:16,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/d8df14e6efe44b178a71f0a259f78b29, entries=150, sequenceid=602, filesize=12.0 K 2024-11-18T20:42:16,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 9fe934dbc79ce3fabcdcf715bb36d192 in 861ms, sequenceid=602, compaction requested=false 2024-11-18T20:42:16,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fe934dbc79ce3fabcdcf715bb36d192: 2024-11-18T20:42:16,809 INFO [Thread-1130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1130. 2024-11-18T20:42:16,809 DEBUG [Thread-1130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:16,809 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:16,810 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:16,810 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:23,989 INFO [Thread-1128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1128. 2024-11-18T20:42:23,989 INFO [Thread-1136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1136. 2024-11-18T20:42:23,989 DEBUG [Thread-1128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:23,989 DEBUG [Thread-1136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:23,989 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:23,989 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:23,989 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:23,989 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:23,990 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:23,990 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:23,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T20:42:23,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-18T20:42:23,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 152 2024-11-18T20:42:23,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 131 2024-11-18T20:42:23,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 198 2024-11-18T20:42:23,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-18T20:42:23,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T20:42:23,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2528 2024-11-18T20:42:23,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2485 2024-11-18T20:42:23,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2447 2024-11-18T20:42:23,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2529 2024-11-18T20:42:23,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2462 2024-11-18T20:42:23,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T20:42:23,990 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:42:23,990 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:42:23,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:23,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:23,990 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:23,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T20:42:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T20:42:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-18T20:42:23,994 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962543994"}]},"ts":"1731962543994"} 2024-11-18T20:42:23,996 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T20:42:23,996 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T20:42:23,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:42:23,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9fe934dbc79ce3fabcdcf715bb36d192, UNASSIGN}] 2024-11-18T20:42:23,999 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9fe934dbc79ce3fabcdcf715bb36d192, UNASSIGN 2024-11-18T20:42:24,000 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=9fe934dbc79ce3fabcdcf715bb36d192, regionState=CLOSING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:24,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9fe934dbc79ce3fabcdcf715bb36d192, UNASSIGN because future has completed 2024-11-18T20:42:24,002 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:42:24,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:42:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-18T20:42:24,155 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:24,155 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:42:24,155 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 9fe934dbc79ce3fabcdcf715bb36d192, disabling compactions & flushes 2024-11-18T20:42:24,155 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:24,155 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:24,155 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. after waiting 0 ms 2024-11-18T20:42:24,155 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:24,155 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing 9fe934dbc79ce3fabcdcf715bb36d192 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-18T20:42:24,156 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=A 2024-11-18T20:42:24,156 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:24,156 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=B 2024-11-18T20:42:24,156 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:24,156 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fe934dbc79ce3fabcdcf715bb36d192, store=C 2024-11-18T20:42:24,156 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:24,161 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/d8fa1036cf864f069cbb32e094251c35 is 50, key is test_row_0/A:col10/1731962543988/Put/seqid=0 2024-11-18T20:42:24,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742172_1348 (size=9857) 2024-11-18T20:42:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-18T20:42:24,566 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=610 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/d8fa1036cf864f069cbb32e094251c35 2024-11-18T20:42:24,576 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/ca99f8a120c0408bb84caeb00d2750bd is 50, key is test_row_0/B:col10/1731962543988/Put/seqid=0 2024-11-18T20:42:24,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742173_1349 (size=9857) 2024-11-18T20:42:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-18T20:42:24,981 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=610 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/ca99f8a120c0408bb84caeb00d2750bd 2024-11-18T20:42:24,989 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/155e3db32fcc4d449aa030d6c777f26a is 50, key is test_row_0/C:col10/1731962543988/Put/seqid=0 2024-11-18T20:42:24,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742174_1350 (size=9857) 2024-11-18T20:42:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-18T20:42:25,394 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=610 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/155e3db32fcc4d449aa030d6c777f26a 2024-11-18T20:42:25,400 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/A/d8fa1036cf864f069cbb32e094251c35 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/d8fa1036cf864f069cbb32e094251c35 2024-11-18T20:42:25,405 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/d8fa1036cf864f069cbb32e094251c35, entries=100, sequenceid=610, filesize=9.6 K 2024-11-18T20:42:25,406 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/B/ca99f8a120c0408bb84caeb00d2750bd as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/ca99f8a120c0408bb84caeb00d2750bd 2024-11-18T20:42:25,410 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/ca99f8a120c0408bb84caeb00d2750bd, entries=100, sequenceid=610, filesize=9.6 K 2024-11-18T20:42:25,411 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/.tmp/C/155e3db32fcc4d449aa030d6c777f26a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/155e3db32fcc4d449aa030d6c777f26a 2024-11-18T20:42:25,414 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/155e3db32fcc4d449aa030d6c777f26a, entries=100, sequenceid=610, filesize=9.6 K 2024-11-18T20:42:25,415 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 9fe934dbc79ce3fabcdcf715bb36d192 in 1260ms, sequenceid=610, compaction requested=true 2024-11-18T20:42:25,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5874b82c9c7841afa3f19ebae8f452d5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/d53d4c15e29f420ea289ef165cc0793f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/3deaedc9f96b45668f7c0dba21f321b1, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/ba223808bb7d4bf9b3f5c9b8eeb82236, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/2bfcd3b96fdd4d68a022a859438278f8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/788417b91c9d4907803eb53ac824c3d0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5b1278e646ed47c88cef248fc2aba868, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/418a8ad1ec18444490f2c29aca0b1cc5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/c34cd99525d04b2d9cf798fca483fab1, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/cdffde8455f84380abddff2d3371769b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/fdf37daff21240f98ebe6dc585a2ce9f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/989472011c184cdca09fe3cf53fb538f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/37a28f5eff6f4237b9bc2ae6ee3fddbe, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/bd35ac5b886c4538b496ae259a8d8b91, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/051b7f7cde1741708cf6851f4f895a07, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/9139f84ea3904919adb68b06e32ded5b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/2cc749f4de6448b39db904a87ccf6206, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/6c37edfceb3a4413a2f6da23b7bd8e74, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/b3cf083785a94c2aa36fa55c98d3fee8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5e85b61a0af44e0fa219c4c4f36e7686, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/290fca8c1a8e484496225917641cb806, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/64c7f396241a4585a884f253132d792d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/396bc5af8b084f7ebe29e778fe9fc618, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/19073208f1ee4b4faf24e8ddac589761, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/e9db33d80ee84168b1053b0cfc614323, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/79f0c93745ff4e08a95d97d35ae39033, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5eca31ac83f4484ea526dd8ba8ccbfe7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/c8c4761498a640419f6936c68f0ee276, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5b21d8dd399d43cb9fe6d0294647ae44, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/739b8bc95d3d4636914b8c5c78a8c236, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/3bf289e95d7b4409a3f1af4409fb6a48, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/7152def1207e4a95aba07afde0057d5a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/56031ec9726441bd8e5f7a20f99b9902] to archive 2024-11-18T20:42:25,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:42:25,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5874b82c9c7841afa3f19ebae8f452d5 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5874b82c9c7841afa3f19ebae8f452d5 2024-11-18T20:42:25,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/d53d4c15e29f420ea289ef165cc0793f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/d53d4c15e29f420ea289ef165cc0793f 2024-11-18T20:42:25,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/3deaedc9f96b45668f7c0dba21f321b1 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/3deaedc9f96b45668f7c0dba21f321b1 2024-11-18T20:42:25,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/ba223808bb7d4bf9b3f5c9b8eeb82236 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/ba223808bb7d4bf9b3f5c9b8eeb82236 2024-11-18T20:42:25,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/2bfcd3b96fdd4d68a022a859438278f8 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/2bfcd3b96fdd4d68a022a859438278f8 2024-11-18T20:42:25,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/788417b91c9d4907803eb53ac824c3d0 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/788417b91c9d4907803eb53ac824c3d0 2024-11-18T20:42:25,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5b1278e646ed47c88cef248fc2aba868 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5b1278e646ed47c88cef248fc2aba868 2024-11-18T20:42:25,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/418a8ad1ec18444490f2c29aca0b1cc5 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/418a8ad1ec18444490f2c29aca0b1cc5 2024-11-18T20:42:25,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/c34cd99525d04b2d9cf798fca483fab1 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/c34cd99525d04b2d9cf798fca483fab1 2024-11-18T20:42:25,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/cdffde8455f84380abddff2d3371769b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/cdffde8455f84380abddff2d3371769b 2024-11-18T20:42:25,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/fdf37daff21240f98ebe6dc585a2ce9f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/fdf37daff21240f98ebe6dc585a2ce9f 2024-11-18T20:42:25,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/989472011c184cdca09fe3cf53fb538f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/989472011c184cdca09fe3cf53fb538f 2024-11-18T20:42:25,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/37a28f5eff6f4237b9bc2ae6ee3fddbe to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/37a28f5eff6f4237b9bc2ae6ee3fddbe 2024-11-18T20:42:25,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/bd35ac5b886c4538b496ae259a8d8b91 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/bd35ac5b886c4538b496ae259a8d8b91 2024-11-18T20:42:25,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/051b7f7cde1741708cf6851f4f895a07 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/051b7f7cde1741708cf6851f4f895a07 2024-11-18T20:42:25,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/9139f84ea3904919adb68b06e32ded5b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/9139f84ea3904919adb68b06e32ded5b 2024-11-18T20:42:25,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/2cc749f4de6448b39db904a87ccf6206 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/2cc749f4de6448b39db904a87ccf6206 2024-11-18T20:42:25,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/6c37edfceb3a4413a2f6da23b7bd8e74 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/6c37edfceb3a4413a2f6da23b7bd8e74 2024-11-18T20:42:25,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/b3cf083785a94c2aa36fa55c98d3fee8 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/b3cf083785a94c2aa36fa55c98d3fee8 2024-11-18T20:42:25,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5e85b61a0af44e0fa219c4c4f36e7686 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5e85b61a0af44e0fa219c4c4f36e7686 2024-11-18T20:42:25,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/290fca8c1a8e484496225917641cb806 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/290fca8c1a8e484496225917641cb806 2024-11-18T20:42:25,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/64c7f396241a4585a884f253132d792d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/64c7f396241a4585a884f253132d792d 2024-11-18T20:42:25,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/396bc5af8b084f7ebe29e778fe9fc618 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/396bc5af8b084f7ebe29e778fe9fc618 2024-11-18T20:42:25,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/19073208f1ee4b4faf24e8ddac589761 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/19073208f1ee4b4faf24e8ddac589761 2024-11-18T20:42:25,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/e9db33d80ee84168b1053b0cfc614323 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/e9db33d80ee84168b1053b0cfc614323 2024-11-18T20:42:25,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/79f0c93745ff4e08a95d97d35ae39033 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/79f0c93745ff4e08a95d97d35ae39033 2024-11-18T20:42:25,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5eca31ac83f4484ea526dd8ba8ccbfe7 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5eca31ac83f4484ea526dd8ba8ccbfe7 2024-11-18T20:42:25,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/c8c4761498a640419f6936c68f0ee276 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/c8c4761498a640419f6936c68f0ee276 2024-11-18T20:42:25,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5b21d8dd399d43cb9fe6d0294647ae44 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/5b21d8dd399d43cb9fe6d0294647ae44 2024-11-18T20:42:25,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/739b8bc95d3d4636914b8c5c78a8c236 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/739b8bc95d3d4636914b8c5c78a8c236 2024-11-18T20:42:25,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/3bf289e95d7b4409a3f1af4409fb6a48 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/3bf289e95d7b4409a3f1af4409fb6a48 2024-11-18T20:42:25,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/7152def1207e4a95aba07afde0057d5a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/7152def1207e4a95aba07afde0057d5a 2024-11-18T20:42:25,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/56031ec9726441bd8e5f7a20f99b9902 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/56031ec9726441bd8e5f7a20f99b9902 2024-11-18T20:42:25,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/53710f3f4bbd460f871d6dc922b95aee, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/f4294cce6a8b468ebafb5e52245f34eb, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/d1f1f02e23a14608b624ed3a5807525c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/3841f76a0ec34973a99dfcdc3cef519c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/5bef8d895756446baf91f8186ac45f4a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/55f5835d9c8142cdad3624cafabe8fe5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/18caf538b4d14e3b85da6243f7b21173, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/a990065f9d9d49f1b5536c09d10fdd2d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/c18e3752dabf45f78e2ab819ab018b8f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/f1dc33de433d4f91bd5be70ad8df7c27, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/0f90af7890dd402397dcab1cbb524715, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/284e7b37477f4d1e9f3e235e9b0e4ec0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/d1d83239ca6b4b079afc76d7e6d19430, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/e48178c2bd7b4b9fb70cdf28407a1610, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/4fcee9c0bc8a4db5a14518b506634c4f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/4dcda4e28a284e65a4996f364b5a8bb0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/b48e96ed22254866896f33e333d4aaff, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/ebc20ad99aa741c5a6e30cf5d5fd57d1, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/d2ce26dcad284f6faaaa3a05e0e1e9a8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/2292448389474fd7888e8b361d9510b4, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/7d2c96a55dc544359c115650a28dd372, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/0c67e950812b4d2f8117e0775e26d8e2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/1f7d13ef2eee4c819bfe01fcd60998bc, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/b841382686fb4b01beaeb38ab1e978a8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/cf3fb5430a654daeac8381f273832fd3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/6d55f52ec1aa49b69fdf355c8357653c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/f9dfd326902d48af95fa8e58b510441c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/ff84483c58da4412a2b59b72164ed135, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/035e26c7277e44febb87070f029f6fd9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/581fdb6985304b66bdab10a22a9289da, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/394cad9551d2467fa46ac5e1ddd36c91, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/fe37f852611f4f0fafddffdf95450f6d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/891ef3f0c32c424bbf9a22d4c6ad6a6d] to archive 2024-11-18T20:42:25,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:42:25,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/53710f3f4bbd460f871d6dc922b95aee to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/53710f3f4bbd460f871d6dc922b95aee 2024-11-18T20:42:25,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/f4294cce6a8b468ebafb5e52245f34eb to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/f4294cce6a8b468ebafb5e52245f34eb 2024-11-18T20:42:25,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/d1f1f02e23a14608b624ed3a5807525c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/d1f1f02e23a14608b624ed3a5807525c 2024-11-18T20:42:25,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/3841f76a0ec34973a99dfcdc3cef519c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/3841f76a0ec34973a99dfcdc3cef519c 2024-11-18T20:42:25,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/5bef8d895756446baf91f8186ac45f4a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/5bef8d895756446baf91f8186ac45f4a 2024-11-18T20:42:25,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/55f5835d9c8142cdad3624cafabe8fe5 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/55f5835d9c8142cdad3624cafabe8fe5 2024-11-18T20:42:25,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/18caf538b4d14e3b85da6243f7b21173 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/18caf538b4d14e3b85da6243f7b21173 2024-11-18T20:42:25,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/a990065f9d9d49f1b5536c09d10fdd2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/a990065f9d9d49f1b5536c09d10fdd2d 2024-11-18T20:42:25,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/c18e3752dabf45f78e2ab819ab018b8f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/c18e3752dabf45f78e2ab819ab018b8f 2024-11-18T20:42:25,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/f1dc33de433d4f91bd5be70ad8df7c27 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/f1dc33de433d4f91bd5be70ad8df7c27 2024-11-18T20:42:25,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/0f90af7890dd402397dcab1cbb524715 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/0f90af7890dd402397dcab1cbb524715 2024-11-18T20:42:25,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/284e7b37477f4d1e9f3e235e9b0e4ec0 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/284e7b37477f4d1e9f3e235e9b0e4ec0 2024-11-18T20:42:25,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/d1d83239ca6b4b079afc76d7e6d19430 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/d1d83239ca6b4b079afc76d7e6d19430 2024-11-18T20:42:25,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/e48178c2bd7b4b9fb70cdf28407a1610 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/e48178c2bd7b4b9fb70cdf28407a1610 2024-11-18T20:42:25,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/4fcee9c0bc8a4db5a14518b506634c4f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/4fcee9c0bc8a4db5a14518b506634c4f 2024-11-18T20:42:25,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/4dcda4e28a284e65a4996f364b5a8bb0 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/4dcda4e28a284e65a4996f364b5a8bb0 2024-11-18T20:42:25,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/b48e96ed22254866896f33e333d4aaff to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/b48e96ed22254866896f33e333d4aaff 2024-11-18T20:42:25,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/ebc20ad99aa741c5a6e30cf5d5fd57d1 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/ebc20ad99aa741c5a6e30cf5d5fd57d1 2024-11-18T20:42:25,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/d2ce26dcad284f6faaaa3a05e0e1e9a8 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/d2ce26dcad284f6faaaa3a05e0e1e9a8 2024-11-18T20:42:25,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/2292448389474fd7888e8b361d9510b4 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/2292448389474fd7888e8b361d9510b4 2024-11-18T20:42:25,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/7d2c96a55dc544359c115650a28dd372 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/7d2c96a55dc544359c115650a28dd372 2024-11-18T20:42:25,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/0c67e950812b4d2f8117e0775e26d8e2 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/0c67e950812b4d2f8117e0775e26d8e2 2024-11-18T20:42:25,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/1f7d13ef2eee4c819bfe01fcd60998bc to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/1f7d13ef2eee4c819bfe01fcd60998bc 2024-11-18T20:42:25,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/b841382686fb4b01beaeb38ab1e978a8 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/b841382686fb4b01beaeb38ab1e978a8 2024-11-18T20:42:25,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/cf3fb5430a654daeac8381f273832fd3 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/cf3fb5430a654daeac8381f273832fd3 2024-11-18T20:42:25,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/6d55f52ec1aa49b69fdf355c8357653c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/6d55f52ec1aa49b69fdf355c8357653c 2024-11-18T20:42:25,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/f9dfd326902d48af95fa8e58b510441c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/f9dfd326902d48af95fa8e58b510441c 2024-11-18T20:42:25,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/ff84483c58da4412a2b59b72164ed135 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/ff84483c58da4412a2b59b72164ed135 2024-11-18T20:42:25,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/035e26c7277e44febb87070f029f6fd9 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/035e26c7277e44febb87070f029f6fd9 2024-11-18T20:42:25,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/581fdb6985304b66bdab10a22a9289da to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/581fdb6985304b66bdab10a22a9289da 2024-11-18T20:42:25,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/394cad9551d2467fa46ac5e1ddd36c91 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/394cad9551d2467fa46ac5e1ddd36c91 2024-11-18T20:42:25,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/fe37f852611f4f0fafddffdf95450f6d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/fe37f852611f4f0fafddffdf95450f6d 2024-11-18T20:42:25,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/891ef3f0c32c424bbf9a22d4c6ad6a6d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/891ef3f0c32c424bbf9a22d4c6ad6a6d 2024-11-18T20:42:25,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/438b93c899804f908691acfbecc471c3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/86d199cd5c3448b690b1d1aa426640e2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/258d4deef73e4ad59ab915f2789f132b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/ea6253327d1d47658d7be106a25f056b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/4305709d24624755af640cf6c960d95f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/666bc5f710e746929e7e34ccfa99a48f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/b4a75ed48dae4e4b85ed2256745be8e3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/6ad902d61eff498b9aebcfabfa72db5d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/537bc733a1f744119ba0cd89300c59a2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/4a3e727d9f09498fb7413ad2df33bd5c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/78bfe400f82b4465877f747108ce9846, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/a2283e6fc59b4ce496d0e378072b3909, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/a1ee03b7e17d465cb0c1ea0094683972, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/2c7dbf3194264ee3b0e383ff8b0e7653, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/09c9e58212984d6d8f79d66a25427584, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/105f91e1cc6a47e3a4b748cb5fb72e7f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/33a8d04053b34e2f90bd5105608da743, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/c8cbc274a6a34f52b442f0c0b411cf43, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/aab26774ddfa4580945f88c0dfda4112, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/d2fd99b4c3a04b4a9674f489826fecc6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/1fe154946a2e46db972fb2e0505af75b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/f76c1f30d57946489b7a5f8beff25288, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/e36f165857194f80b810cce51606af1e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/858227cd71394efe8aa7b2be19d78940, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/c3400a0973bb4688b622e9d092683cf2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/43b8b9a1265d43fe9658a74616c8e15e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/edcea59a73124d87aeb1865d7c1fbb06, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/01af5ee4199842fb9d43864d8838721d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/0b3a5b59ca1b4aa891eda176fed4705a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/706d5d17c3e249eca8a0d34bb90e176b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/41c6feb8a66e43078fd2cee0e2ba165c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/fbf1f935c7054558879c5703a97622a1, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/70cc9453a325406bbf70d812defd66a0] to archive 2024-11-18T20:42:25,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:42:25,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/438b93c899804f908691acfbecc471c3 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/438b93c899804f908691acfbecc471c3 2024-11-18T20:42:25,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/86d199cd5c3448b690b1d1aa426640e2 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/86d199cd5c3448b690b1d1aa426640e2 2024-11-18T20:42:25,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/258d4deef73e4ad59ab915f2789f132b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/258d4deef73e4ad59ab915f2789f132b 2024-11-18T20:42:25,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/ea6253327d1d47658d7be106a25f056b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/ea6253327d1d47658d7be106a25f056b 2024-11-18T20:42:25,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/4305709d24624755af640cf6c960d95f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/4305709d24624755af640cf6c960d95f 2024-11-18T20:42:25,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/666bc5f710e746929e7e34ccfa99a48f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/666bc5f710e746929e7e34ccfa99a48f 2024-11-18T20:42:25,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/b4a75ed48dae4e4b85ed2256745be8e3 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/b4a75ed48dae4e4b85ed2256745be8e3 2024-11-18T20:42:25,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/6ad902d61eff498b9aebcfabfa72db5d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/6ad902d61eff498b9aebcfabfa72db5d 2024-11-18T20:42:25,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/537bc733a1f744119ba0cd89300c59a2 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/537bc733a1f744119ba0cd89300c59a2 2024-11-18T20:42:25,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/4a3e727d9f09498fb7413ad2df33bd5c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/4a3e727d9f09498fb7413ad2df33bd5c 2024-11-18T20:42:25,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/78bfe400f82b4465877f747108ce9846 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/78bfe400f82b4465877f747108ce9846 2024-11-18T20:42:25,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/a2283e6fc59b4ce496d0e378072b3909 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/a2283e6fc59b4ce496d0e378072b3909 2024-11-18T20:42:25,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/a1ee03b7e17d465cb0c1ea0094683972 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/a1ee03b7e17d465cb0c1ea0094683972 2024-11-18T20:42:25,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/2c7dbf3194264ee3b0e383ff8b0e7653 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/2c7dbf3194264ee3b0e383ff8b0e7653 2024-11-18T20:42:25,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/09c9e58212984d6d8f79d66a25427584 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/09c9e58212984d6d8f79d66a25427584 2024-11-18T20:42:25,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/105f91e1cc6a47e3a4b748cb5fb72e7f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/105f91e1cc6a47e3a4b748cb5fb72e7f 2024-11-18T20:42:25,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/33a8d04053b34e2f90bd5105608da743 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/33a8d04053b34e2f90bd5105608da743 2024-11-18T20:42:25,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/c8cbc274a6a34f52b442f0c0b411cf43 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/c8cbc274a6a34f52b442f0c0b411cf43 2024-11-18T20:42:25,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/aab26774ddfa4580945f88c0dfda4112 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/aab26774ddfa4580945f88c0dfda4112 2024-11-18T20:42:25,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/d2fd99b4c3a04b4a9674f489826fecc6 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/d2fd99b4c3a04b4a9674f489826fecc6 2024-11-18T20:42:25,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/1fe154946a2e46db972fb2e0505af75b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/1fe154946a2e46db972fb2e0505af75b 2024-11-18T20:42:25,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/f76c1f30d57946489b7a5f8beff25288 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/f76c1f30d57946489b7a5f8beff25288 2024-11-18T20:42:25,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/e36f165857194f80b810cce51606af1e to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/e36f165857194f80b810cce51606af1e 2024-11-18T20:42:25,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/858227cd71394efe8aa7b2be19d78940 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/858227cd71394efe8aa7b2be19d78940 2024-11-18T20:42:25,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/c3400a0973bb4688b622e9d092683cf2 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/c3400a0973bb4688b622e9d092683cf2 2024-11-18T20:42:25,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/43b8b9a1265d43fe9658a74616c8e15e to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/43b8b9a1265d43fe9658a74616c8e15e 2024-11-18T20:42:25,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/edcea59a73124d87aeb1865d7c1fbb06 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/edcea59a73124d87aeb1865d7c1fbb06 2024-11-18T20:42:25,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/01af5ee4199842fb9d43864d8838721d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/01af5ee4199842fb9d43864d8838721d 2024-11-18T20:42:25,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/0b3a5b59ca1b4aa891eda176fed4705a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/0b3a5b59ca1b4aa891eda176fed4705a 2024-11-18T20:42:25,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/706d5d17c3e249eca8a0d34bb90e176b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/706d5d17c3e249eca8a0d34bb90e176b 2024-11-18T20:42:25,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/41c6feb8a66e43078fd2cee0e2ba165c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/41c6feb8a66e43078fd2cee0e2ba165c 2024-11-18T20:42:25,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/fbf1f935c7054558879c5703a97622a1 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/fbf1f935c7054558879c5703a97622a1 2024-11-18T20:42:25,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/70cc9453a325406bbf70d812defd66a0 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/70cc9453a325406bbf70d812defd66a0 2024-11-18T20:42:25,498 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/recovered.edits/613.seqid, newMaxSeqId=613, maxSeqId=1 2024-11-18T20:42:25,498 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. 2024-11-18T20:42:25,498 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 9fe934dbc79ce3fabcdcf715bb36d192: Waiting for close lock at 1731962544155Running coprocessor pre-close hooks at 1731962544155Disabling compacts and flushes for region at 1731962544155Disabling writes for close at 1731962544155Obtaining lock to block concurrent updates at 1731962544155Preparing flush snapshotting stores in 9fe934dbc79ce3fabcdcf715bb36d192 at 1731962544156 (+1 ms)Finished memstore snapshotting TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731962544156Flushing stores of TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192. at 1731962544157 (+1 ms)Flushing 9fe934dbc79ce3fabcdcf715bb36d192/A: creating writer at 1731962544157Flushing 9fe934dbc79ce3fabcdcf715bb36d192/A: appending metadata at 1731962544160 (+3 ms)Flushing 9fe934dbc79ce3fabcdcf715bb36d192/A: closing flushed file at 1731962544160Flushing 9fe934dbc79ce3fabcdcf715bb36d192/B: creating writer at 1731962544572 (+412 ms)Flushing 9fe934dbc79ce3fabcdcf715bb36d192/B: appending metadata at 1731962544575 (+3 ms)Flushing 9fe934dbc79ce3fabcdcf715bb36d192/B: closing flushed file at 1731962544575Flushing 9fe934dbc79ce3fabcdcf715bb36d192/C: creating writer at 1731962544985 (+410 ms)Flushing 9fe934dbc79ce3fabcdcf715bb36d192/C: appending metadata at 1731962544989 (+4 ms)Flushing 9fe934dbc79ce3fabcdcf715bb36d192/C: closing flushed file at 1731962544989Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69ee5a8e: reopening flushed file at 1731962545399 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cffb9f6: reopening flushed file at 1731962545405 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60b039ec: reopening flushed file at 1731962545410 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 9fe934dbc79ce3fabcdcf715bb36d192 in 1260ms, sequenceid=610, compaction requested=true at 1731962545415 (+5 ms)Writing region close event to WAL at 1731962545495 (+80 ms)Running coprocessor post-close hooks at 1731962545498 (+3 ms)Closed at 1731962545498 2024-11-18T20:42:25,499 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:25,500 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=9fe934dbc79ce3fabcdcf715bb36d192, regionState=CLOSED 2024-11-18T20:42:25,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 because future has completed 2024-11-18T20:42:25,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-18T20:42:25,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 9fe934dbc79ce3fabcdcf715bb36d192, server=c0a89b2656d4,45393,1731962440107 in 1.4990 sec 2024-11-18T20:42:25,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-18T20:42:25,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9fe934dbc79ce3fabcdcf715bb36d192, UNASSIGN in 1.5050 sec 2024-11-18T20:42:25,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-18T20:42:25,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5080 sec 2024-11-18T20:42:25,506 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962545506"}]},"ts":"1731962545506"} 2024-11-18T20:42:25,507 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T20:42:25,507 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T20:42:25,508 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5170 sec 2024-11-18T20:42:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-18T20:42:26,138 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:26,139 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T20:42:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:42:26,142 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:42:26,143 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:42:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-18T20:42:26,147 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:26,150 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A, FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B, FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C, FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/recovered.edits] 2024-11-18T20:42:26,153 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/01121a6318804143994c313788cbfe59 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/01121a6318804143994c313788cbfe59 2024-11-18T20:42:26,155 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/1a1d1813ad314c29923a1b599195a6bb to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/1a1d1813ad314c29923a1b599195a6bb 2024-11-18T20:42:26,157 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/d8fa1036cf864f069cbb32e094251c35 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/A/d8fa1036cf864f069cbb32e094251c35 2024-11-18T20:42:26,160 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/973a9c712f6c4dc1af1f27bc56c2649b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/973a9c712f6c4dc1af1f27bc56c2649b 2024-11-18T20:42:26,162 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/ca99f8a120c0408bb84caeb00d2750bd to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/ca99f8a120c0408bb84caeb00d2750bd 2024-11-18T20:42:26,163 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/e76cfea2e1314fa0ac94955b75383083 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/B/e76cfea2e1314fa0ac94955b75383083 2024-11-18T20:42:26,166 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/155e3db32fcc4d449aa030d6c777f26a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/155e3db32fcc4d449aa030d6c777f26a 2024-11-18T20:42:26,168 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/5f98c0af3fae4767a252b16cb59465c1 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/5f98c0af3fae4767a252b16cb59465c1 2024-11-18T20:42:26,169 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/d8df14e6efe44b178a71f0a259f78b29 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/C/d8df14e6efe44b178a71f0a259f78b29 2024-11-18T20:42:26,172 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/recovered.edits/613.seqid to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192/recovered.edits/613.seqid 2024-11-18T20:42:26,172 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/9fe934dbc79ce3fabcdcf715bb36d192 2024-11-18T20:42:26,172 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T20:42:26,174 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:42:26,177 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T20:42:26,179 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T20:42:26,180 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:42:26,180 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T20:42:26,180 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731962546180"}]},"ts":"9223372036854775807"} 2024-11-18T20:42:26,182 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T20:42:26,182 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9fe934dbc79ce3fabcdcf715bb36d192, NAME => 'TestAcidGuarantees,,1731962513401.9fe934dbc79ce3fabcdcf715bb36d192.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T20:42:26,183 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T20:42:26,183 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731962546183"}]},"ts":"9223372036854775807"} 2024-11-18T20:42:26,185 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T20:42:26,186 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:42:26,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-18T20:42:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-18T20:42:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T20:42:26,248 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:26,264 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 248), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=389 (was 371) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1612 (was 1882) 2024-11-18T20:42:26,272 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=389, ProcessCount=11, AvailableMemoryMB=1612 2024-11-18T20:42:26,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:42:26,273 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:42:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T20:42:26,275 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T20:42:26,275 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:26,276 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-18T20:42:26,276 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T20:42:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T20:42:26,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742175_1351 (size=1061) 2024-11-18T20:42:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T20:42:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T20:42:26,685 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093 2024-11-18T20:42:26,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742176_1352 (size=53) 2024-11-18T20:42:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T20:42:27,096 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:42:27,097 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 75114f181d047350ddda984d66f6e223, disabling compactions & flushes 2024-11-18T20:42:27,097 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:27,097 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:27,097 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. after waiting 0 ms 2024-11-18T20:42:27,097 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:27,097 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:27,097 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 75114f181d047350ddda984d66f6e223: Waiting for close lock at 1731962547097Disabling compacts and flushes for region at 1731962547097Disabling writes for close at 1731962547097Writing region close event to WAL at 1731962547097Closed at 1731962547097 2024-11-18T20:42:27,099 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T20:42:27,100 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731962547099"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731962547099"}]},"ts":"1731962547099"} 2024-11-18T20:42:27,104 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T20:42:27,106 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T20:42:27,106 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962547106"}]},"ts":"1731962547106"} 2024-11-18T20:42:27,108 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T20:42:27,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=75114f181d047350ddda984d66f6e223, ASSIGN}] 2024-11-18T20:42:27,109 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=75114f181d047350ddda984d66f6e223, ASSIGN 2024-11-18T20:42:27,110 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=75114f181d047350ddda984d66f6e223, ASSIGN; state=OFFLINE, location=c0a89b2656d4,45393,1731962440107; forceNewPlan=false, retain=false 2024-11-18T20:42:27,261 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=75114f181d047350ddda984d66f6e223, regionState=OPENING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:27,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=75114f181d047350ddda984d66f6e223, ASSIGN because future has completed 2024-11-18T20:42:27,264 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:42:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T20:42:27,423 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:27,423 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:42:27,423 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:27,424 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:42:27,424 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:27,424 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:27,426 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:27,428 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:42:27,429 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75114f181d047350ddda984d66f6e223 columnFamilyName A 2024-11-18T20:42:27,429 DEBUG [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:27,430 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] regionserver.HStore(327): Store=75114f181d047350ddda984d66f6e223/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:42:27,431 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:27,433 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:42:27,433 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75114f181d047350ddda984d66f6e223 columnFamilyName B 2024-11-18T20:42:27,433 DEBUG [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:27,434 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] regionserver.HStore(327): Store=75114f181d047350ddda984d66f6e223/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:42:27,434 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:27,436 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:42:27,437 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75114f181d047350ddda984d66f6e223 columnFamilyName C 2024-11-18T20:42:27,437 DEBUG [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:27,437 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] regionserver.HStore(327): Store=75114f181d047350ddda984d66f6e223/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:42:27,437 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:27,437 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:27,438 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223 2024-11-18T20:42:27,438 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223 2024-11-18T20:42:27,439 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:27,439 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:27,439 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:42:27,440 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:27,442 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:42:27,442 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened 75114f181d047350ddda984d66f6e223; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69728333, jitterRate=0.039033129811286926}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:42:27,442 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:27,443 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for 75114f181d047350ddda984d66f6e223: Running coprocessor pre-open hook at 1731962547424Writing region info on filesystem at 1731962547424Initializing all the Stores at 1731962547425 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962547425Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962547426 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962547426Cleaning up temporary data from old regions at 1731962547439 (+13 ms)Running coprocessor post-open hooks at 1731962547442 (+3 ms)Region opened successfully at 1731962547443 (+1 ms) 2024-11-18T20:42:27,443 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., pid=91, masterSystemTime=1731962547418 2024-11-18T20:42:27,445 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:27,445 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:27,445 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=75114f181d047350ddda984d66f6e223, regionState=OPEN, openSeqNum=2, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:27,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 because future has completed 2024-11-18T20:42:27,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-18T20:42:27,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 in 183 msec 2024-11-18T20:42:27,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-18T20:42:27,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=75114f181d047350ddda984d66f6e223, ASSIGN in 341 msec 2024-11-18T20:42:27,451 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T20:42:27,451 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962547451"}]},"ts":"1731962547451"} 2024-11-18T20:42:27,452 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T20:42:27,452 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T20:42:27,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-11-18T20:42:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T20:42:28,429 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:28,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d493646, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,431 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:42:28,431 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:42:28,432 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:42:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:42:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:42:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@377afd8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:42:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:42:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:28,435 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:42:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71575f45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,436 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:42:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:42:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:42:28,438 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:42:28,439 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:42:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c0a89b2656d4,46825,1731962439484 2024-11-18T20:42:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@676f26f5 2024-11-18T20:42:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:42:28,440 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:42:28,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:42:28,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:42:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-18T20:42:28,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742177_1353 (size=1097) 2024-11-18T20:42:28,452 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-18T20:42:28,452 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-18T20:42:28,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:42:28,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=75114f181d047350ddda984d66f6e223, REOPEN/MOVE}] 2024-11-18T20:42:28,457 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=75114f181d047350ddda984d66f6e223, REOPEN/MOVE 2024-11-18T20:42:28,458 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=75114f181d047350ddda984d66f6e223, regionState=CLOSING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:28,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=75114f181d047350ddda984d66f6e223, REOPEN/MOVE because future has completed 2024-11-18T20:42:28,459 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:42:28,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:42:28,612 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:28,612 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:42:28,612 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 75114f181d047350ddda984d66f6e223, disabling compactions & flushes 2024-11-18T20:42:28,612 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:28,612 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:28,612 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. after waiting 0 ms 2024-11-18T20:42:28,612 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:28,616 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T20:42:28,617 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:28,617 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 75114f181d047350ddda984d66f6e223: Waiting for close lock at 1731962548612Running coprocessor pre-close hooks at 1731962548612Disabling compacts and flushes for region at 1731962548612Disabling writes for close at 1731962548612Writing region close event to WAL at 1731962548612Running coprocessor post-close hooks at 1731962548617 (+5 ms)Closed at 1731962548617 2024-11-18T20:42:28,617 WARN [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: 75114f181d047350ddda984d66f6e223 to self. 2024-11-18T20:42:28,619 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:28,620 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=75114f181d047350ddda984d66f6e223, regionState=CLOSED 2024-11-18T20:42:28,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 because future has completed 2024-11-18T20:42:28,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-18T20:42:28,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 in 164 msec 2024-11-18T20:42:28,625 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=75114f181d047350ddda984d66f6e223, REOPEN/MOVE; state=CLOSED, location=c0a89b2656d4,45393,1731962440107; forceNewPlan=false, retain=true 2024-11-18T20:42:28,776 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=75114f181d047350ddda984d66f6e223, regionState=OPENING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:28,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=75114f181d047350ddda984d66f6e223, REOPEN/MOVE because future has completed 2024-11-18T20:42:28,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:42:28,935 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:28,935 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:42:28,936 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:28,936 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:42:28,936 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:28,936 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:28,938 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:28,939 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:42:28,940 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75114f181d047350ddda984d66f6e223 columnFamilyName A 2024-11-18T20:42:28,941 DEBUG [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:28,941 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] regionserver.HStore(327): Store=75114f181d047350ddda984d66f6e223/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:42:28,942 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:28,942 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:42:28,943 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75114f181d047350ddda984d66f6e223 columnFamilyName B 2024-11-18T20:42:28,943 DEBUG [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:28,943 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] regionserver.HStore(327): Store=75114f181d047350ddda984d66f6e223/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:42:28,943 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:28,944 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:42:28,944 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75114f181d047350ddda984d66f6e223 columnFamilyName C 2024-11-18T20:42:28,944 DEBUG [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:28,944 INFO [StoreOpener-75114f181d047350ddda984d66f6e223-1 {}] regionserver.HStore(327): Store=75114f181d047350ddda984d66f6e223/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:42:28,944 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:28,944 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:28,945 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223 2024-11-18T20:42:28,946 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223 2024-11-18T20:42:28,947 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:28,947 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:28,947 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:42:28,948 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:28,949 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened 75114f181d047350ddda984d66f6e223; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66824860, jitterRate=-0.00423198938369751}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:42:28,949 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:28,949 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for 75114f181d047350ddda984d66f6e223: Running coprocessor pre-open hook at 1731962548936Writing region info on filesystem at 1731962548936Initializing all the Stores at 1731962548937 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962548938 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962548938Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962548938Cleaning up temporary data from old regions at 1731962548947 (+9 ms)Running coprocessor post-open hooks at 1731962548949 (+2 ms)Region opened successfully at 1731962548949 2024-11-18T20:42:28,950 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., pid=96, masterSystemTime=1731962548930 2024-11-18T20:42:28,952 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:28,952 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:28,952 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=75114f181d047350ddda984d66f6e223, regionState=OPEN, openSeqNum=5, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:28,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 because future has completed 2024-11-18T20:42:28,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-18T20:42:28,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 in 176 msec 2024-11-18T20:42:28,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-18T20:42:28,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=75114f181d047350ddda984d66f6e223, REOPEN/MOVE in 499 msec 2024-11-18T20:42:28,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-18T20:42:28,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 504 msec 2024-11-18T20:42:28,962 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 518 msec 2024-11-18T20:42:28,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-18T20:42:28,963 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:28,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c6bde19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,963 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:42:28,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:42:28,964 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:42:28,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:42:28,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:42:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@240cebca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:42:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:42:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:28,966 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:42:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c3112, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@173aa1ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,967 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:42:28,967 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:42:28,968 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:42:28,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:42:28,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:42:28,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21faf3a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:42:28,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:42:28,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:28,970 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:42:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c8afbaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d5e8400, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,971 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:42:28,971 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:42:28,972 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:42:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:42:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:42:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bbc92e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:42:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:42:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:28,974 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:42:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@abc6b29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ce5c6bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:42:28,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:42:28,977 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:42:28,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:42:28,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:42:28,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@742909ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:42:28,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:42:28,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:28,979 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:42:28,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@263a1df3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7fbdbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:42:28,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:42:28,981 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:42:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:42:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:42:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25da3358, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:42:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:42:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:28,982 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:42:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f10c141, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69edc398, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,984 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:42:28,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:42:28,984 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:42:28,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:42:28,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:42:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4c3e75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:42:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:42:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:28,986 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:42:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6707c2bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bbf44a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,986 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:42:28,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:42:28,987 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:42:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:42:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:42:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@798c10f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:42:28,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:42:28,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:28,988 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:42:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22133bd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ad32ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,989 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:42:28,989 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:42:28,990 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:42:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:42:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:42:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10dd455c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:42:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:42:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:28,991 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:42:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60ecd4bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@615de503, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,993 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:42:28,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:42:28,994 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:42:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:42:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:42:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@480c00de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:42:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:42:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:28,996 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:42:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71dabe74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5690e3aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,997 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:42:28,998 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:42:28,998 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:42:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:42:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:42:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75848937, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:42:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:42:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:29,001 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:42:29,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@194816d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:42:29,002 DEBUG [Thread-1621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:42:29,002 DEBUG [Thread-1623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:42:29,002 DEBUG [Thread-1619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:42:29,002 DEBUG [Thread-1625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:42:29,002 DEBUG [Thread-1610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:42:29,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:42:29,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:42:29,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:42:29,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:42:29,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:42:29,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:42:29,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:42:29,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:42:29,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:42:29,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:42:29,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:42:29,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:42:29,005 DEBUG [Thread-1612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:42:29,005 DEBUG [Thread-1606 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:42:29,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:42:29,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:42:29,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:42:29,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:42:29,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:42:29,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:42:29,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:42:29,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:42:29,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:42:29,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:42:29,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:42:29,008 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:42:29,008 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:42:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:42:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:42:29,010 DEBUG [Thread-1617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:42:29,010 DEBUG [Thread-1614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:42:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-18T20:42:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:42:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:42:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:42:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:42:29,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T20:42:29,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:42:29,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:42:29,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:42:29,013 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:42:29,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:42:29,013 DEBUG [Thread-1608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:42:29,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:42:29,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:42:29,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:42:29,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:42:29,016 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:42:29,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:42:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:42:29,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:42:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:29,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 75114f181d047350ddda984d66f6e223 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T20:42:29,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=A 2024-11-18T20:42:29,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:29,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=B 2024-11-18T20:42:29,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:29,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=C 2024-11-18T20:42:29,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:29,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187a6487fa1d274e93a87982179e0997ba_75114f181d047350ddda984d66f6e223 is 50, key is test_row_0/A:col10/1731962549024/Put/seqid=0 2024-11-18T20:42:29,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742178_1354 (size=14594) 2024-11-18T20:42:29,061 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:29,064 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187a6487fa1d274e93a87982179e0997ba_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187a6487fa1d274e93a87982179e0997ba_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:29,065 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/5d7f8261d02f4068a67fad1a9c77b59a, store: [table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:29,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/5d7f8261d02f4068a67fad1a9c77b59a is 175, key is test_row_0/A:col10/1731962549024/Put/seqid=0 2024-11-18T20:42:29,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962609061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731962549056, max timestamp=1731962549069Num uniques 859; 2024-11-18T20:42:29,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731962549056, max timestamp=1731962549069Num uniques 859; 2024-11-18T20:42:29,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T20:42:29,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T20:42:29,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731962549056, max timestamp=1731962549069Num uniques 859; 2024-11-18T20:42:29,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T20:42:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742179_1355 (size=39549) 2024-11-18T20:42:29,093 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/5d7f8261d02f4068a67fad1a9c77b59a 2024-11-18T20:42:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T20:42:29,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/d9dab9b1df1f45e2831d8ef068c62e86 is 50, key is test_row_0/B:col10/1731962549024/Put/seqid=0 2024-11-18T20:42:29,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742180_1356 (size=12001) 2024-11-18T20:42:29,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962609118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43860 deadline: 1731962609120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43882 deadline: 1731962609124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1731962609125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T20:42:29,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:29,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:29,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:29,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:29,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:29,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962609188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962609237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43860 deadline: 1731962609237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43882 deadline: 1731962609247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1731962609247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T20:42:29,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:29,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:29,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:29,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:29,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T20:42:29,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962609407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962609457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43860 deadline: 1731962609457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43882 deadline: 1731962609458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1731962609458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T20:42:29,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:29,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:29,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:29,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:29,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:29,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/d9dab9b1df1f45e2831d8ef068c62e86 2024-11-18T20:42:29,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/d3d0597c149a478da9794511e8fee9ba is 50, key is test_row_0/C:col10/1731962549024/Put/seqid=0 2024-11-18T20:42:29,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742181_1357 (size=12001) 2024-11-18T20:42:29,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T20:42:29,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:29,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:29,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:29,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:29,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T20:42:29,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962609717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962609768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43860 deadline: 1731962609768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T20:42:29,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:29,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:29,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:29,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:29,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:29,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43882 deadline: 1731962609777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:29,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1731962609777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:29,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T20:42:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:29,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:29,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/d3d0597c149a478da9794511e8fee9ba 2024-11-18T20:42:29,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/5d7f8261d02f4068a67fad1a9c77b59a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/5d7f8261d02f4068a67fad1a9c77b59a 2024-11-18T20:42:29,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/5d7f8261d02f4068a67fad1a9c77b59a, entries=200, sequenceid=15, filesize=38.6 K 2024-11-18T20:42:29,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/d9dab9b1df1f45e2831d8ef068c62e86 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/d9dab9b1df1f45e2831d8ef068c62e86 2024-11-18T20:42:29,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/d9dab9b1df1f45e2831d8ef068c62e86, entries=150, sequenceid=15, filesize=11.7 K 2024-11-18T20:42:29,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/d3d0597c149a478da9794511e8fee9ba as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/d3d0597c149a478da9794511e8fee9ba 2024-11-18T20:42:29,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/d3d0597c149a478da9794511e8fee9ba, entries=150, sequenceid=15, filesize=11.7 K 2024-11-18T20:42:29,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 75114f181d047350ddda984d66f6e223 in 941ms, sequenceid=15, compaction requested=false 2024-11-18T20:42:29,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:30,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T20:42:30,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:30,088 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 75114f181d047350ddda984d66f6e223 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-18T20:42:30,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=A 2024-11-18T20:42:30,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:42:30,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=B 2024-11-18T20:42:30,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:42:30,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=C 2024-11-18T20:42:30,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:42:30,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ee96bf45da204aaeabd4829382fbb569_75114f181d047350ddda984d66f6e223 is 50, key is test_row_0/A:col10/1731962549123/Put/seqid=0 2024-11-18T20:42:30,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742182_1358 (size=12154) 2024-11-18T20:42:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T20:42:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:30,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:30,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43860 deadline: 1731962610287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43882 deadline: 1731962610287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:30,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1731962610287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:30,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962610297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:30,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:30,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962610298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:30,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:30,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962610417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:30,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962610427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:30,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:30,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:30,507 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ee96bf45da204aaeabd4829382fbb569_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ee96bf45da204aaeabd4829382fbb569_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:30,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/d4f052ad08ba4a0aae4e579f64415451, store: [table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:30,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/d4f052ad08ba4a0aae4e579f64415451 is 175, key is test_row_0/A:col10/1731962549123/Put/seqid=0 2024-11-18T20:42:30,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742183_1359 (size=30955) 2024-11-18T20:42:30,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:30,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962610637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:30,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:30,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962610647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:30,932 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/d4f052ad08ba4a0aae4e579f64415451 2024-11-18T20:42:30,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/5809ea51939844d5a4a1858b572312b6 is 50, key is test_row_0/B:col10/1731962549123/Put/seqid=0 2024-11-18T20:42:30,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742184_1360 (size=12001) 2024-11-18T20:42:30,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962610957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:30,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962610957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:31,038 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T20:42:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T20:42:31,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43882 deadline: 1731962611307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:31,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1731962611307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:31,317 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 75114f181d047350ddda984d66f6e223 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:42:31,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:31,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:31,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:31,318 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 75114f181d047350ddda984d66f6e223 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:42:31,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:31,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:31,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:31,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43860 deadline: 1731962611317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:31,327 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in 75114f181d047350ddda984d66f6e223 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:42:31,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:31,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:31,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:31,343 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/5809ea51939844d5a4a1858b572312b6 2024-11-18T20:42:31,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/f55207773a2b458baac51b8eb2956c8b is 50, key is test_row_0/C:col10/1731962549123/Put/seqid=0 2024-11-18T20:42:31,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742185_1361 (size=12001) 2024-11-18T20:42:31,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962611477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:31,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:31,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:31,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:31,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962611477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:31,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:31,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:31,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:31,760 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/f55207773a2b458baac51b8eb2956c8b 2024-11-18T20:42:31,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/d4f052ad08ba4a0aae4e579f64415451 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/d4f052ad08ba4a0aae4e579f64415451 2024-11-18T20:42:31,771 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/d4f052ad08ba4a0aae4e579f64415451, entries=150, sequenceid=49, filesize=30.2 K 2024-11-18T20:42:31,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/5809ea51939844d5a4a1858b572312b6 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/5809ea51939844d5a4a1858b572312b6 2024-11-18T20:42:31,777 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/5809ea51939844d5a4a1858b572312b6, entries=150, sequenceid=49, filesize=11.7 K 2024-11-18T20:42:31,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/f55207773a2b458baac51b8eb2956c8b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/f55207773a2b458baac51b8eb2956c8b 2024-11-18T20:42:31,786 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/f55207773a2b458baac51b8eb2956c8b, entries=150, sequenceid=49, filesize=11.7 K 2024-11-18T20:42:31,787 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 75114f181d047350ddda984d66f6e223 in 1699ms, sequenceid=49, compaction requested=false 2024-11-18T20:42:31,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:31,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:31,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-18T20:42:31,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-18T20:42:31,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-18T20:42:31,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7730 sec 2024-11-18T20:42:31,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.7850 sec 2024-11-18T20:42:32,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 75114f181d047350ddda984d66f6e223 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T20:42:32,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=A 2024-11-18T20:42:32,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:32,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=B 2024-11-18T20:42:32,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:32,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=C 2024-11-18T20:42:32,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:32,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185f2a68cd719843bd8bbd7460b1128aa7_75114f181d047350ddda984d66f6e223 is 50, key is test_row_0/A:col10/1731962550296/Put/seqid=0 2024-11-18T20:42:32,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742186_1362 (size=14594) 2024-11-18T20:42:32,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:32,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962612599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:32,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:32,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962612599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:32,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962612717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:32,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962612717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:32,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:32,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:32,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:32,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:32,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:32,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:32,928 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:32,932 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185f2a68cd719843bd8bbd7460b1128aa7_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185f2a68cd719843bd8bbd7460b1128aa7_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:32,933 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/415cb70e89154894af87ccf60864a167, store: [table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:32,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/415cb70e89154894af87ccf60864a167 is 175, key is test_row_0/A:col10/1731962550296/Put/seqid=0 2024-11-18T20:42:32,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962612927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:32,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742187_1363 (size=39549) 2024-11-18T20:42:32,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962612937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:32,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:32,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:32,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:33,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T20:42:33,167 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:33,168 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:42:33,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-18T20:42:33,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:42:33,171 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:42:33,171 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:42:33,172 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:42:33,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:33,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962613247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:33,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:33,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:33,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962613258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:33,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:42:33,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:42:33,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:33,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:33,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:33,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:33,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:33,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:33,342 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/415cb70e89154894af87ccf60864a167 2024-11-18T20:42:33,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:33,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1731962613338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:33,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:33,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43882 deadline: 1731962613338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:33,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:33,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43860 deadline: 1731962613338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:33,346 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 75114f181d047350ddda984d66f6e223 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:42:33,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:33,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:33,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:33,347 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in 75114f181d047350ddda984d66f6e223 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:42:33,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:33,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:33,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:33,348 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 75114f181d047350ddda984d66f6e223 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:42:33,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:33,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:33,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:33,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/ec528ecd9da34d70a9f90be6dd4591f1 is 50, key is test_row_0/B:col10/1731962550296/Put/seqid=0 2024-11-18T20:42:33,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742188_1364 (size=12001) 2024-11-18T20:42:33,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:42:33,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:33,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:33,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:33,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:33,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:42:33,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:42:33,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:33,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:33,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:33,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:33,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:33,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:33,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/ec528ecd9da34d70a9f90be6dd4591f1 2024-11-18T20:42:33,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:33,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962613757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:33,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:33,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:33,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:33,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/3568da97debb4da2954bb0559c7ffc6c is 50, key is test_row_0/C:col10/1731962550296/Put/seqid=0 2024-11-18T20:42:33,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742189_1365 (size=12001) 2024-11-18T20:42:33,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:42:33,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:33,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:33,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:33,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:33,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:33,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962613777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:42:33,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:42:33,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:33,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:33,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:33,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:33,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:34,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:42:34,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:34,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:34,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:34,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:34,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:34,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:34,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/3568da97debb4da2954bb0559c7ffc6c 2024-11-18T20:42:34,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/415cb70e89154894af87ccf60864a167 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/415cb70e89154894af87ccf60864a167 2024-11-18T20:42:34,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/415cb70e89154894af87ccf60864a167, entries=200, sequenceid=61, filesize=38.6 K 2024-11-18T20:42:34,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/ec528ecd9da34d70a9f90be6dd4591f1 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/ec528ecd9da34d70a9f90be6dd4591f1 2024-11-18T20:42:34,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/ec528ecd9da34d70a9f90be6dd4591f1, entries=150, sequenceid=61, filesize=11.7 K 2024-11-18T20:42:34,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/3568da97debb4da2954bb0559c7ffc6c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/3568da97debb4da2954bb0559c7ffc6c 2024-11-18T20:42:34,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/3568da97debb4da2954bb0559c7ffc6c, entries=150, sequenceid=61, filesize=11.7 K 2024-11-18T20:42:34,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 75114f181d047350ddda984d66f6e223 in 1689ms, sequenceid=61, compaction requested=true 2024-11-18T20:42:34,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:34,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75114f181d047350ddda984d66f6e223:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:42:34,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:34,192 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:34,192 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:34,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75114f181d047350ddda984d66f6e223:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:42:34,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:34,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75114f181d047350ddda984d66f6e223:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:42:34,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:34,195 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:34,195 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 75114f181d047350ddda984d66f6e223/B is initiating minor compaction (all files) 2024-11-18T20:42:34,195 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75114f181d047350ddda984d66f6e223/B in TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:34,195 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/d9dab9b1df1f45e2831d8ef068c62e86, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/5809ea51939844d5a4a1858b572312b6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/ec528ecd9da34d70a9f90be6dd4591f1] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp, totalSize=35.2 K 2024-11-18T20:42:34,195 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:34,195 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting d9dab9b1df1f45e2831d8ef068c62e86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731962549016 2024-11-18T20:42:34,195 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 75114f181d047350ddda984d66f6e223/A is initiating minor compaction (all files) 2024-11-18T20:42:34,195 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75114f181d047350ddda984d66f6e223/A in TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:34,195 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/5d7f8261d02f4068a67fad1a9c77b59a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/d4f052ad08ba4a0aae4e579f64415451, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/415cb70e89154894af87ccf60864a167] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp, totalSize=107.5 K 2024-11-18T20:42:34,196 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:34,196 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/5d7f8261d02f4068a67fad1a9c77b59a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/d4f052ad08ba4a0aae4e579f64415451, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/415cb70e89154894af87ccf60864a167] 2024-11-18T20:42:34,196 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 5809ea51939844d5a4a1858b572312b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731962549118 2024-11-18T20:42:34,196 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d7f8261d02f4068a67fad1a9c77b59a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731962549016 2024-11-18T20:42:34,196 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting ec528ecd9da34d70a9f90be6dd4591f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731962550282 2024-11-18T20:42:34,196 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4f052ad08ba4a0aae4e579f64415451, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731962549118 2024-11-18T20:42:34,197 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 415cb70e89154894af87ccf60864a167, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731962550282 2024-11-18T20:42:34,206 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75114f181d047350ddda984d66f6e223#B#compaction#314 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:34,206 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/9f31885e21e646728fca455f1292284a is 50, key is test_row_0/B:col10/1731962550296/Put/seqid=0 2024-11-18T20:42:34,210 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:34,221 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411185e0e4659961249fea83b6c9f252e6e7e_75114f181d047350ddda984d66f6e223 store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:34,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742190_1366 (size=12104) 2024-11-18T20:42:34,225 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411185e0e4659961249fea83b6c9f252e6e7e_75114f181d047350ddda984d66f6e223, store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:34,225 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185e0e4659961249fea83b6c9f252e6e7e_75114f181d047350ddda984d66f6e223 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:34,230 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/9f31885e21e646728fca455f1292284a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/9f31885e21e646728fca455f1292284a 2024-11-18T20:42:34,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742191_1367 (size=4469) 2024-11-18T20:42:34,237 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75114f181d047350ddda984d66f6e223/B of 75114f181d047350ddda984d66f6e223 into 9f31885e21e646728fca455f1292284a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:34,237 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:34,237 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., storeName=75114f181d047350ddda984d66f6e223/B, priority=13, startTime=1731962554192; duration=0sec 2024-11-18T20:42:34,238 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:34,238 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75114f181d047350ddda984d66f6e223:B 2024-11-18T20:42:34,238 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:34,238 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:34,239 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 75114f181d047350ddda984d66f6e223/C is initiating minor compaction (all files) 2024-11-18T20:42:34,239 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75114f181d047350ddda984d66f6e223/C in TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:34,239 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/d3d0597c149a478da9794511e8fee9ba, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/f55207773a2b458baac51b8eb2956c8b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/3568da97debb4da2954bb0559c7ffc6c] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp, totalSize=35.2 K 2024-11-18T20:42:34,239 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting d3d0597c149a478da9794511e8fee9ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731962549016 2024-11-18T20:42:34,239 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting f55207773a2b458baac51b8eb2956c8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731962549118 2024-11-18T20:42:34,240 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 3568da97debb4da2954bb0559c7ffc6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731962550282 2024-11-18T20:42:34,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:42:34,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:34,241 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 75114f181d047350ddda984d66f6e223 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T20:42:34,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=A 2024-11-18T20:42:34,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:34,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=B 2024-11-18T20:42:34,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:34,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=C 2024-11-18T20:42:34,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:34,251 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75114f181d047350ddda984d66f6e223#C#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:34,252 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/2bd69bbd5f6448088a7d883d7c5cd1e7 is 50, key is test_row_0/C:col10/1731962550296/Put/seqid=0 2024-11-18T20:42:34,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118933c18f1237a429c902e69fa0f66480e_75114f181d047350ddda984d66f6e223 is 50, key is test_row_0/A:col10/1731962552585/Put/seqid=0 2024-11-18T20:42:34,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742192_1368 (size=12104) 2024-11-18T20:42:34,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742193_1369 (size=12154) 2024-11-18T20:42:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:42:34,637 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75114f181d047350ddda984d66f6e223#A#compaction#315 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:34,637 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/11d866565ba546eb9281cd7b90875ecd is 175, key is test_row_0/A:col10/1731962550296/Put/seqid=0 2024-11-18T20:42:34,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742194_1370 (size=31058) 2024-11-18T20:42:34,676 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/2bd69bbd5f6448088a7d883d7c5cd1e7 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/2bd69bbd5f6448088a7d883d7c5cd1e7 2024-11-18T20:42:34,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:34,681 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118933c18f1237a429c902e69fa0f66480e_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118933c18f1237a429c902e69fa0f66480e_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:34,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/592c05c72cd0417a815520d0869e7ca0, store: [table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:34,682 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75114f181d047350ddda984d66f6e223/C of 75114f181d047350ddda984d66f6e223 into 2bd69bbd5f6448088a7d883d7c5cd1e7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:34,682 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:34,682 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., storeName=75114f181d047350ddda984d66f6e223/C, priority=13, startTime=1731962554192; duration=0sec 2024-11-18T20:42:34,682 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:34,682 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75114f181d047350ddda984d66f6e223:C 2024-11-18T20:42:34,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/592c05c72cd0417a815520d0869e7ca0 is 175, key is test_row_0/A:col10/1731962552585/Put/seqid=0 2024-11-18T20:42:34,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742195_1371 (size=30955) 2024-11-18T20:42:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:34,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:34,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962614827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:34,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962614828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:34,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962614947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:34,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962614947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:35,046 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/11d866565ba546eb9281cd7b90875ecd as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/11d866565ba546eb9281cd7b90875ecd 2024-11-18T20:42:35,050 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75114f181d047350ddda984d66f6e223/A of 75114f181d047350ddda984d66f6e223 into 11d866565ba546eb9281cd7b90875ecd(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:35,050 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:35,050 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., storeName=75114f181d047350ddda984d66f6e223/A, priority=13, startTime=1731962554192; duration=0sec 2024-11-18T20:42:35,050 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:35,050 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75114f181d047350ddda984d66f6e223:A 2024-11-18T20:42:35,095 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/592c05c72cd0417a815520d0869e7ca0 2024-11-18T20:42:35,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/bc269f474fae4d9088c863ece66f1c49 is 50, key is test_row_0/B:col10/1731962552585/Put/seqid=0 2024-11-18T20:42:35,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742196_1372 (size=12001) 2024-11-18T20:42:35,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962615157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:35,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962615157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:42:35,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962615467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:35,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:35,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:35,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962615467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:35,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:35,512 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/bc269f474fae4d9088c863ece66f1c49 2024-11-18T20:42:35,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/49b181f62d404adbbe248e89e5c20b5b is 50, key is test_row_0/C:col10/1731962552585/Put/seqid=0 2024-11-18T20:42:35,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742197_1373 (size=12001) 2024-11-18T20:42:35,928 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/49b181f62d404adbbe248e89e5c20b5b 2024-11-18T20:42:35,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/592c05c72cd0417a815520d0869e7ca0 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/592c05c72cd0417a815520d0869e7ca0 2024-11-18T20:42:35,936 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/592c05c72cd0417a815520d0869e7ca0, entries=150, sequenceid=82, filesize=30.2 K 2024-11-18T20:42:35,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/bc269f474fae4d9088c863ece66f1c49 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/bc269f474fae4d9088c863ece66f1c49 2024-11-18T20:42:35,941 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/bc269f474fae4d9088c863ece66f1c49, entries=150, sequenceid=82, filesize=11.7 K 2024-11-18T20:42:35,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/49b181f62d404adbbe248e89e5c20b5b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/49b181f62d404adbbe248e89e5c20b5b 2024-11-18T20:42:35,946 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/49b181f62d404adbbe248e89e5c20b5b, entries=150, sequenceid=82, filesize=11.7 K 2024-11-18T20:42:35,947 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 75114f181d047350ddda984d66f6e223 in 1705ms, sequenceid=82, compaction requested=false 2024-11-18T20:42:35,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:35,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:35,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-18T20:42:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-18T20:42:35,950 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-18T20:42:35,950 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7770 sec 2024-11-18T20:42:35,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.7820 sec 2024-11-18T20:42:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:35,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 75114f181d047350ddda984d66f6e223 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T20:42:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=A 2024-11-18T20:42:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=B 2024-11-18T20:42:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=C 2024-11-18T20:42:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:35,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c5c29028fa0c44fca944b673e59e4820_75114f181d047350ddda984d66f6e223 is 50, key is test_row_0/A:col10/1731962554825/Put/seqid=0 2024-11-18T20:42:36,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742198_1374 (size=14594) 2024-11-18T20:42:36,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962616084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:36,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962616090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:36,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:36,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962616207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:36,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:36,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:36,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:36,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962616217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:36,407 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:36,411 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c5c29028fa0c44fca944b673e59e4820_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c5c29028fa0c44fca944b673e59e4820_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:36,412 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/dbc7b96bf2694e58b2bcc8a4289864fa, store: [table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:36,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/dbc7b96bf2694e58b2bcc8a4289864fa is 175, key is test_row_0/A:col10/1731962554825/Put/seqid=0 2024-11-18T20:42:36,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742199_1375 (size=39549) 2024-11-18T20:42:36,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962616427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:36,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962616437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:36,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962616737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:36,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:36,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:36,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:36,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962616757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:36,819 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/dbc7b96bf2694e58b2bcc8a4289864fa 2024-11-18T20:42:36,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/8aae9cc896c9464281a31489982c2ddc is 50, key is test_row_0/B:col10/1731962554825/Put/seqid=0 2024-11-18T20:42:36,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742200_1376 (size=12001) 2024-11-18T20:42:37,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/8aae9cc896c9464281a31489982c2ddc 2024-11-18T20:42:37,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/8a15d7e691954aa684ac54802ccde0d4 is 50, key is test_row_0/C:col10/1731962554825/Put/seqid=0 2024-11-18T20:42:37,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742201_1377 (size=12001) 2024-11-18T20:42:37,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962617257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:37,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962617277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:37,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:37,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:37,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:42:37,327 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:37,328 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:42:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-18T20:42:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T20:42:37,329 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:42:37,330 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:42:37,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:42:37,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1731962617357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:37,364 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 75114f181d047350ddda984d66f6e223 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:42:37,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:37,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:37,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:37,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43882 deadline: 1731962617377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:37,381 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 75114f181d047350ddda984d66f6e223 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:42:37,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:37,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:37,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:37,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43860 deadline: 1731962617397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:37,404 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in 75114f181d047350ddda984d66f6e223 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8283 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:42:37,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:37,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:37,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T20:42:37,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-18T20:42:37,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:37,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:37,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:37,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:37,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:37,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-18T20:42:37,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:37,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:37,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:37,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:37,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:37,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T20:42:37,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/8a15d7e691954aa684ac54802ccde0d4 2024-11-18T20:42:37,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/dbc7b96bf2694e58b2bcc8a4289864fa as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/dbc7b96bf2694e58b2bcc8a4289864fa 2024-11-18T20:42:37,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/dbc7b96bf2694e58b2bcc8a4289864fa, entries=200, sequenceid=97, filesize=38.6 K 2024-11-18T20:42:37,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/8aae9cc896c9464281a31489982c2ddc as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/8aae9cc896c9464281a31489982c2ddc 2024-11-18T20:42:37,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/8aae9cc896c9464281a31489982c2ddc, entries=150, sequenceid=97, filesize=11.7 K 2024-11-18T20:42:37,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/8a15d7e691954aa684ac54802ccde0d4 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/8a15d7e691954aa684ac54802ccde0d4 2024-11-18T20:42:37,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/8a15d7e691954aa684ac54802ccde0d4, entries=150, sequenceid=97, filesize=11.7 K 2024-11-18T20:42:37,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 75114f181d047350ddda984d66f6e223 in 1675ms, sequenceid=97, compaction requested=true 2024-11-18T20:42:37,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:37,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75114f181d047350ddda984d66f6e223:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:42:37,664 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:37,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:37,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75114f181d047350ddda984d66f6e223:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:42:37,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:37,665 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:37,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75114f181d047350ddda984d66f6e223:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:42:37,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:37,665 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:37,665 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 75114f181d047350ddda984d66f6e223/A is initiating minor compaction (all files) 2024-11-18T20:42:37,665 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75114f181d047350ddda984d66f6e223/A in TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:37,665 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:37,666 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 75114f181d047350ddda984d66f6e223/B is initiating minor compaction (all files) 2024-11-18T20:42:37,666 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/11d866565ba546eb9281cd7b90875ecd, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/592c05c72cd0417a815520d0869e7ca0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/dbc7b96bf2694e58b2bcc8a4289864fa] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp, totalSize=99.2 K 2024-11-18T20:42:37,666 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:37,666 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75114f181d047350ddda984d66f6e223/B in TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:37,666 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/11d866565ba546eb9281cd7b90875ecd, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/592c05c72cd0417a815520d0869e7ca0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/dbc7b96bf2694e58b2bcc8a4289864fa] 2024-11-18T20:42:37,666 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/9f31885e21e646728fca455f1292284a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/bc269f474fae4d9088c863ece66f1c49, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/8aae9cc896c9464281a31489982c2ddc] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp, totalSize=35.3 K 2024-11-18T20:42:37,666 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11d866565ba546eb9281cd7b90875ecd, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731962550282 2024-11-18T20:42:37,666 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f31885e21e646728fca455f1292284a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731962550282 2024-11-18T20:42:37,666 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 592c05c72cd0417a815520d0869e7ca0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731962552585 2024-11-18T20:42:37,666 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting bc269f474fae4d9088c863ece66f1c49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731962552585 2024-11-18T20:42:37,666 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbc7b96bf2694e58b2bcc8a4289864fa, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731962554798 2024-11-18T20:42:37,667 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aae9cc896c9464281a31489982c2ddc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731962554825 2024-11-18T20:42:37,671 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:37,672 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411189a16939b2b204da9b5288183f8284ecb_75114f181d047350ddda984d66f6e223 store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:37,673 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75114f181d047350ddda984d66f6e223#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:37,674 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411189a16939b2b204da9b5288183f8284ecb_75114f181d047350ddda984d66f6e223, store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:37,674 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189a16939b2b204da9b5288183f8284ecb_75114f181d047350ddda984d66f6e223 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:37,674 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/6896a8b97a8c4042ab523a04140e4ca0 is 50, key is test_row_0/B:col10/1731962554825/Put/seqid=0 2024-11-18T20:42:37,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742202_1378 (size=12207) 2024-11-18T20:42:37,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742203_1379 (size=4469) 2024-11-18T20:42:37,685 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75114f181d047350ddda984d66f6e223#A#compaction#323 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:37,685 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/b6b321f935d7423dbf20a544be5fe0fe is 175, key is test_row_0/A:col10/1731962554825/Put/seqid=0 2024-11-18T20:42:37,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742204_1380 (size=31161) 2024-11-18T20:42:37,693 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/b6b321f935d7423dbf20a544be5fe0fe as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/b6b321f935d7423dbf20a544be5fe0fe 2024-11-18T20:42:37,697 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75114f181d047350ddda984d66f6e223/A of 75114f181d047350ddda984d66f6e223 into b6b321f935d7423dbf20a544be5fe0fe(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:37,697 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:37,697 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., storeName=75114f181d047350ddda984d66f6e223/A, priority=13, startTime=1731962557664; duration=0sec 2024-11-18T20:42:37,697 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:37,697 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75114f181d047350ddda984d66f6e223:A 2024-11-18T20:42:37,697 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:37,698 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:37,698 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 75114f181d047350ddda984d66f6e223/C is initiating minor compaction (all files) 2024-11-18T20:42:37,698 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75114f181d047350ddda984d66f6e223/C in TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:37,698 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/2bd69bbd5f6448088a7d883d7c5cd1e7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/49b181f62d404adbbe248e89e5c20b5b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/8a15d7e691954aa684ac54802ccde0d4] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp, totalSize=35.3 K 2024-11-18T20:42:37,698 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bd69bbd5f6448088a7d883d7c5cd1e7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731962550282 2024-11-18T20:42:37,699 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49b181f62d404adbbe248e89e5c20b5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731962552585 2024-11-18T20:42:37,699 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a15d7e691954aa684ac54802ccde0d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731962554825 2024-11-18T20:42:37,703 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75114f181d047350ddda984d66f6e223#C#compaction#325 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:37,704 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/e8d1d4c5cd7f4a69a48335772cb84198 is 50, key is test_row_0/C:col10/1731962554825/Put/seqid=0 2024-11-18T20:42:37,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742205_1381 (size=12207) 2024-11-18T20:42:37,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-18T20:42:37,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:37,787 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 75114f181d047350ddda984d66f6e223 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T20:42:37,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=A 2024-11-18T20:42:37,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:37,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=B 2024-11-18T20:42:37,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:37,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=C 2024-11-18T20:42:37,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:37,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183c039d83f7ca420c81ac1508ea0794ab_75114f181d047350ddda984d66f6e223 is 50, key is test_row_0/A:col10/1731962556082/Put/seqid=0 2024-11-18T20:42:37,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742206_1382 (size=12154) 2024-11-18T20:42:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T20:42:38,081 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/6896a8b97a8c4042ab523a04140e4ca0 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/6896a8b97a8c4042ab523a04140e4ca0 2024-11-18T20:42:38,084 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75114f181d047350ddda984d66f6e223/B of 75114f181d047350ddda984d66f6e223 into 6896a8b97a8c4042ab523a04140e4ca0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:38,084 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:38,085 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., storeName=75114f181d047350ddda984d66f6e223/B, priority=13, startTime=1731962557665; duration=0sec 2024-11-18T20:42:38,085 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:38,085 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75114f181d047350ddda984d66f6e223:B 2024-11-18T20:42:38,114 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/e8d1d4c5cd7f4a69a48335772cb84198 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/e8d1d4c5cd7f4a69a48335772cb84198 2024-11-18T20:42:38,118 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75114f181d047350ddda984d66f6e223/C of 75114f181d047350ddda984d66f6e223 into e8d1d4c5cd7f4a69a48335772cb84198(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:38,118 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:38,118 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., storeName=75114f181d047350ddda984d66f6e223/C, priority=13, startTime=1731962557665; duration=0sec 2024-11-18T20:42:38,118 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:38,118 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75114f181d047350ddda984d66f6e223:C 2024-11-18T20:42:38,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:38,198 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183c039d83f7ca420c81ac1508ea0794ab_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183c039d83f7ca420c81ac1508ea0794ab_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:38,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/072dc5c27c1b49418fb46fe726159f36, store: [table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:38,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/072dc5c27c1b49418fb46fe726159f36 is 175, key is test_row_0/A:col10/1731962556082/Put/seqid=0 2024-11-18T20:42:38,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742207_1383 (size=30955) 2024-11-18T20:42:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:38,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:38,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962618314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:38,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962618315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:38,409 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:42:38,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962618427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:38,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:38,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:38,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962618427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:38,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:38,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:38,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:38,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T20:42:38,603 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/072dc5c27c1b49418fb46fe726159f36 2024-11-18T20:42:38,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/eef0fa88f88d4b93ae145d070eb771c9 is 50, key is test_row_0/B:col10/1731962556082/Put/seqid=0 2024-11-18T20:42:38,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742208_1384 (size=12001) 2024-11-18T20:42:38,612 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/eef0fa88f88d4b93ae145d070eb771c9 2024-11-18T20:42:38,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/505e2eb448694f9590a06a0d52483736 is 50, key is test_row_0/C:col10/1731962556082/Put/seqid=0 2024-11-18T20:42:38,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742209_1385 (size=12001) 2024-11-18T20:42:38,629 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/505e2eb448694f9590a06a0d52483736 2024-11-18T20:42:38,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/072dc5c27c1b49418fb46fe726159f36 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/072dc5c27c1b49418fb46fe726159f36 2024-11-18T20:42:38,637 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/072dc5c27c1b49418fb46fe726159f36, entries=150, sequenceid=117, filesize=30.2 K 2024-11-18T20:42:38,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/eef0fa88f88d4b93ae145d070eb771c9 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/eef0fa88f88d4b93ae145d070eb771c9 2024-11-18T20:42:38,641 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/eef0fa88f88d4b93ae145d070eb771c9, entries=150, sequenceid=117, filesize=11.7 K 2024-11-18T20:42:38,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/505e2eb448694f9590a06a0d52483736 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/505e2eb448694f9590a06a0d52483736 2024-11-18T20:42:38,645 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/505e2eb448694f9590a06a0d52483736, entries=150, sequenceid=117, filesize=11.7 K 2024-11-18T20:42:38,645 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 75114f181d047350ddda984d66f6e223 in 859ms, sequenceid=117, compaction requested=false 2024-11-18T20:42:38,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:38,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:38,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-18T20:42:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-18T20:42:38,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-18T20:42:38,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3170 sec 2024-11-18T20:42:38,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.3210 sec 2024-11-18T20:42:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:38,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 75114f181d047350ddda984d66f6e223 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T20:42:38,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=A 2024-11-18T20:42:38,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:38,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=B 2024-11-18T20:42:38,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:38,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=C 2024-11-18T20:42:38,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:38,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118208969f0e9ae4bcda1a8ce67193396bf_75114f181d047350ddda984d66f6e223 is 50, key is test_row_0/A:col10/1731962558650/Put/seqid=0 2024-11-18T20:42:38,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742210_1386 (size=14794) 2024-11-18T20:42:38,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962618719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:38,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962618722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:38,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:38,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:38,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:38,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962618827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:38,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:38,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:38,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:38,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962618837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:39,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962619048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:39,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962619048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:39,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:39,062 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:39,065 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118208969f0e9ae4bcda1a8ce67193396bf_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118208969f0e9ae4bcda1a8ce67193396bf_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:39,066 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/c7386bbe16d8424f9f5032e0071bcf74, store: [table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:39,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/c7386bbe16d8424f9f5032e0071bcf74 is 175, key is test_row_0/A:col10/1731962558650/Put/seqid=0 2024-11-18T20:42:39,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742211_1387 (size=39749) 2024-11-18T20:42:39,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962619357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:39,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962619357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:39,471 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/c7386bbe16d8424f9f5032e0071bcf74 2024-11-18T20:42:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T20:42:39,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/87e3fee24642415594829f4d2e41fce8 is 50, key is test_row_0/B:col10/1731962558650/Put/seqid=0 2024-11-18T20:42:39,478 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:39,478 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:42:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-18T20:42:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:42:39,482 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:42:39,483 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:42:39,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:42:39,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742212_1388 (size=12151) 2024-11-18T20:42:39,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/87e3fee24642415594829f4d2e41fce8 2024-11-18T20:42:39,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/8747d700335748998febe0f539c23f5c is 50, key is test_row_0/C:col10/1731962558650/Put/seqid=0 2024-11-18T20:42:39,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742213_1389 (size=12151) 2024-11-18T20:42:39,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/8747d700335748998febe0f539c23f5c 2024-11-18T20:42:39,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/c7386bbe16d8424f9f5032e0071bcf74 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/c7386bbe16d8424f9f5032e0071bcf74 2024-11-18T20:42:39,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/c7386bbe16d8424f9f5032e0071bcf74, entries=200, sequenceid=133, filesize=38.8 K 2024-11-18T20:42:39,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/87e3fee24642415594829f4d2e41fce8 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/87e3fee24642415594829f4d2e41fce8 2024-11-18T20:42:39,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/87e3fee24642415594829f4d2e41fce8, entries=150, sequenceid=133, filesize=11.9 K 2024-11-18T20:42:39,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/8747d700335748998febe0f539c23f5c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/8747d700335748998febe0f539c23f5c 2024-11-18T20:42:39,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/8747d700335748998febe0f539c23f5c, entries=150, sequenceid=133, filesize=11.9 K 2024-11-18T20:42:39,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 75114f181d047350ddda984d66f6e223 in 883ms, sequenceid=133, compaction requested=true 2024-11-18T20:42:39,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:39,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75114f181d047350ddda984d66f6e223:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:42:39,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:39,533 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:39,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75114f181d047350ddda984d66f6e223:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:42:39,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:39,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75114f181d047350ddda984d66f6e223:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:42:39,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:39,534 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:39,534 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:39,534 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 75114f181d047350ddda984d66f6e223/A is initiating minor compaction (all files) 2024-11-18T20:42:39,534 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75114f181d047350ddda984d66f6e223/A in TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:39,534 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/b6b321f935d7423dbf20a544be5fe0fe, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/072dc5c27c1b49418fb46fe726159f36, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/c7386bbe16d8424f9f5032e0071bcf74] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp, totalSize=99.5 K 2024-11-18T20:42:39,534 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:39,534 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/b6b321f935d7423dbf20a544be5fe0fe, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/072dc5c27c1b49418fb46fe726159f36, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/c7386bbe16d8424f9f5032e0071bcf74] 2024-11-18T20:42:39,535 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6b321f935d7423dbf20a544be5fe0fe, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731962554825 2024-11-18T20:42:39,538 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:39,538 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 75114f181d047350ddda984d66f6e223/B is initiating minor compaction (all files) 2024-11-18T20:42:39,538 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75114f181d047350ddda984d66f6e223/B in TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:39,538 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/6896a8b97a8c4042ab523a04140e4ca0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/eef0fa88f88d4b93ae145d070eb771c9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/87e3fee24642415594829f4d2e41fce8] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp, totalSize=35.5 K 2024-11-18T20:42:39,538 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 072dc5c27c1b49418fb46fe726159f36, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731962556041 2024-11-18T20:42:39,539 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 6896a8b97a8c4042ab523a04140e4ca0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731962554825 2024-11-18T20:42:39,539 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7386bbe16d8424f9f5032e0071bcf74, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731962558308 2024-11-18T20:42:39,539 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting eef0fa88f88d4b93ae145d070eb771c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731962556041 2024-11-18T20:42:39,539 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 87e3fee24642415594829f4d2e41fce8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731962558313 2024-11-18T20:42:39,549 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:39,552 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75114f181d047350ddda984d66f6e223#B#compaction#332 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:39,553 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118fbd04fed17f749b59ba7aac58aa40aa3_75114f181d047350ddda984d66f6e223 store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:39,554 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/f86dded809554e45adee1600175094a3 is 50, key is test_row_0/B:col10/1731962558650/Put/seqid=0 2024-11-18T20:42:39,555 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118fbd04fed17f749b59ba7aac58aa40aa3_75114f181d047350ddda984d66f6e223, store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:39,555 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118fbd04fed17f749b59ba7aac58aa40aa3_75114f181d047350ddda984d66f6e223 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:39,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742214_1390 (size=12459) 2024-11-18T20:42:39,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742215_1391 (size=4469) 2024-11-18T20:42:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:42:39,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T20:42:39,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:39,636 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 75114f181d047350ddda984d66f6e223 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T20:42:39,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=A 2024-11-18T20:42:39,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:39,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=B 2024-11-18T20:42:39,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:39,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=C 2024-11-18T20:42:39,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:39,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118939afbf848bd46869eadb9623735f412_75114f181d047350ddda984d66f6e223 is 50, key is test_row_0/A:col10/1731962558692/Put/seqid=0 2024-11-18T20:42:39,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742216_1392 (size=12304) 2024-11-18T20:42:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:42:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:39,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:39,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962619919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:39,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:39,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:39,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962619920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:39,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:39,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:39,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:39,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:39,970 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/f86dded809554e45adee1600175094a3 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/f86dded809554e45adee1600175094a3 2024-11-18T20:42:39,975 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75114f181d047350ddda984d66f6e223/B of 75114f181d047350ddda984d66f6e223 into f86dded809554e45adee1600175094a3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:39,975 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:39,975 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., storeName=75114f181d047350ddda984d66f6e223/B, priority=13, startTime=1731962559533; duration=0sec 2024-11-18T20:42:39,975 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:39,975 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75114f181d047350ddda984d66f6e223:B 2024-11-18T20:42:39,975 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:39,976 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:39,976 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 75114f181d047350ddda984d66f6e223/C is initiating minor compaction (all files) 2024-11-18T20:42:39,976 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75114f181d047350ddda984d66f6e223/C in TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:39,976 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/e8d1d4c5cd7f4a69a48335772cb84198, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/505e2eb448694f9590a06a0d52483736, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/8747d700335748998febe0f539c23f5c] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp, totalSize=35.5 K 2024-11-18T20:42:39,976 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75114f181d047350ddda984d66f6e223#A#compaction#333 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:39,976 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting e8d1d4c5cd7f4a69a48335772cb84198, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731962554825 2024-11-18T20:42:39,977 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/7018b79192f84d83828f229186bfa5ec is 175, key is test_row_0/A:col10/1731962558650/Put/seqid=0 2024-11-18T20:42:39,977 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 505e2eb448694f9590a06a0d52483736, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731962556041 2024-11-18T20:42:39,977 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 8747d700335748998febe0f539c23f5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731962558313 2024-11-18T20:42:39,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742217_1393 (size=31413) 2024-11-18T20:42:39,985 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75114f181d047350ddda984d66f6e223#C#compaction#335 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:39,985 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/9fa384e089dc4cb98293b0af929e5fc0 is 50, key is test_row_0/C:col10/1731962558650/Put/seqid=0 2024-11-18T20:42:39,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742218_1394 (size=12459) 2024-11-18T20:42:40,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:40,051 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118939afbf848bd46869eadb9623735f412_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118939afbf848bd46869eadb9623735f412_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/678e6e94a5b84435b353de460c8a4998, store: [table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/678e6e94a5b84435b353de460c8a4998 is 175, key is test_row_0/A:col10/1731962558692/Put/seqid=0 2024-11-18T20:42:40,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742219_1395 (size=31105) 2024-11-18T20:42:40,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962620047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:40,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962620048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:40,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:40,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:40,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:40,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:42:40,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962620267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:40,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:40,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962620267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:40,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:40,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:40,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:40,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:40,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:40,384 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/7018b79192f84d83828f229186bfa5ec as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/7018b79192f84d83828f229186bfa5ec 2024-11-18T20:42:40,388 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75114f181d047350ddda984d66f6e223/A of 75114f181d047350ddda984d66f6e223 into 7018b79192f84d83828f229186bfa5ec(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:40,388 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:40,388 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., storeName=75114f181d047350ddda984d66f6e223/A, priority=13, startTime=1731962559533; duration=0sec 2024-11-18T20:42:40,388 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:40,388 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75114f181d047350ddda984d66f6e223:A 2024-11-18T20:42:40,393 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/9fa384e089dc4cb98293b0af929e5fc0 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/9fa384e089dc4cb98293b0af929e5fc0 2024-11-18T20:42:40,396 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75114f181d047350ddda984d66f6e223/C of 75114f181d047350ddda984d66f6e223 into 9fa384e089dc4cb98293b0af929e5fc0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:40,396 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:40,396 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., storeName=75114f181d047350ddda984d66f6e223/C, priority=13, startTime=1731962559533; duration=0sec 2024-11-18T20:42:40,396 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:40,396 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75114f181d047350ddda984d66f6e223:C 2024-11-18T20:42:40,455 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/678e6e94a5b84435b353de460c8a4998 2024-11-18T20:42:40,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/7456076980cf4c67a985b22aa5bf6fe2 is 50, key is test_row_0/B:col10/1731962558692/Put/seqid=0 2024-11-18T20:42:40,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742220_1396 (size=12151) 2024-11-18T20:42:40,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962620587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:40,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962620587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:40,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:42:40,865 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/7456076980cf4c67a985b22aa5bf6fe2 2024-11-18T20:42:40,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/1f2309c80c4141acaefb1a2140fc3f01 is 50, key is test_row_0/C:col10/1731962558692/Put/seqid=0 2024-11-18T20:42:40,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742221_1397 (size=12151) 2024-11-18T20:42:41,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962621107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:41,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:41,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:41,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:41,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962621107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:41,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:41,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:41,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:41,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:41,276 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/1f2309c80c4141acaefb1a2140fc3f01 2024-11-18T20:42:41,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/678e6e94a5b84435b353de460c8a4998 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/678e6e94a5b84435b353de460c8a4998 2024-11-18T20:42:41,284 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/678e6e94a5b84435b353de460c8a4998, entries=150, sequenceid=151, filesize=30.4 K 2024-11-18T20:42:41,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/7456076980cf4c67a985b22aa5bf6fe2 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/7456076980cf4c67a985b22aa5bf6fe2 2024-11-18T20:42:41,288 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/7456076980cf4c67a985b22aa5bf6fe2, entries=150, sequenceid=151, filesize=11.9 K 2024-11-18T20:42:41,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/1f2309c80c4141acaefb1a2140fc3f01 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/1f2309c80c4141acaefb1a2140fc3f01 2024-11-18T20:42:41,293 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/1f2309c80c4141acaefb1a2140fc3f01, entries=150, sequenceid=151, filesize=11.9 K 2024-11-18T20:42:41,293 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 75114f181d047350ddda984d66f6e223 in 1657ms, sequenceid=151, compaction requested=false 2024-11-18T20:42:41,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:41,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:41,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-18T20:42:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-18T20:42:41,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-18T20:42:41,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8120 sec 2024-11-18T20:42:41,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.8190 sec 2024-11-18T20:42:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:42:41,627 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:41,628 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:42:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-18T20:42:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:42:41,629 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:42:41,630 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:42:41,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:42:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:42:41,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T20:42:41,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:41,782 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 75114f181d047350ddda984d66f6e223 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T20:42:41,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=A 2024-11-18T20:42:41,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:41,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=B 2024-11-18T20:42:41,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:41,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=C 2024-11-18T20:42:41,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:41,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189cb3fc4823444f588be33c304231e581_75114f181d047350ddda984d66f6e223 is 50, key is test_row_0/A:col10/1731962559907/Put/seqid=0 2024-11-18T20:42:41,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742222_1398 (size=12304) 2024-11-18T20:42:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:42:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:42,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:42,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:42,199 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189cb3fc4823444f588be33c304231e581_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189cb3fc4823444f588be33c304231e581_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:42,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/af388636adf64e24a63b1a759fd389eb, store: [table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:42,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/af388636adf64e24a63b1a759fd389eb is 175, key is test_row_0/A:col10/1731962559907/Put/seqid=0 2024-11-18T20:42:42,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742223_1399 (size=31105) 2024-11-18T20:42:42,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962622195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:42,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:42,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:42,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:42,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:42,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962622197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:42,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:42,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:42,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:42,239 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/5d7f8261d02f4068a67fad1a9c77b59a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/d4f052ad08ba4a0aae4e579f64415451, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/415cb70e89154894af87ccf60864a167, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/11d866565ba546eb9281cd7b90875ecd, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/592c05c72cd0417a815520d0869e7ca0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/dbc7b96bf2694e58b2bcc8a4289864fa, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/b6b321f935d7423dbf20a544be5fe0fe, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/072dc5c27c1b49418fb46fe726159f36, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/c7386bbe16d8424f9f5032e0071bcf74] to archive 2024-11-18T20:42:42,239 WARN [regionserver/c0a89b2656d4:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-18T20:42:42,240 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:42:42,242 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/5d7f8261d02f4068a67fad1a9c77b59a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/5d7f8261d02f4068a67fad1a9c77b59a 2024-11-18T20:42:42,244 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/d4f052ad08ba4a0aae4e579f64415451 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/d4f052ad08ba4a0aae4e579f64415451 2024-11-18T20:42:42,246 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/415cb70e89154894af87ccf60864a167 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/415cb70e89154894af87ccf60864a167 2024-11-18T20:42:42,247 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/11d866565ba546eb9281cd7b90875ecd to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/11d866565ba546eb9281cd7b90875ecd 2024-11-18T20:42:42,248 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/592c05c72cd0417a815520d0869e7ca0 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/592c05c72cd0417a815520d0869e7ca0 2024-11-18T20:42:42,248 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/dbc7b96bf2694e58b2bcc8a4289864fa to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/dbc7b96bf2694e58b2bcc8a4289864fa 2024-11-18T20:42:42,253 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/b6b321f935d7423dbf20a544be5fe0fe to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/b6b321f935d7423dbf20a544be5fe0fe 2024-11-18T20:42:42,255 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/072dc5c27c1b49418fb46fe726159f36 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/072dc5c27c1b49418fb46fe726159f36 2024-11-18T20:42:42,256 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/c7386bbe16d8424f9f5032e0071bcf74 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/c7386bbe16d8424f9f5032e0071bcf74 2024-11-18T20:42:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:42:42,258 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/d9dab9b1df1f45e2831d8ef068c62e86, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/5809ea51939844d5a4a1858b572312b6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/9f31885e21e646728fca455f1292284a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/ec528ecd9da34d70a9f90be6dd4591f1, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/bc269f474fae4d9088c863ece66f1c49, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/6896a8b97a8c4042ab523a04140e4ca0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/8aae9cc896c9464281a31489982c2ddc, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/eef0fa88f88d4b93ae145d070eb771c9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/87e3fee24642415594829f4d2e41fce8] to archive 2024-11-18T20:42:42,259 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:42:42,261 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/d9dab9b1df1f45e2831d8ef068c62e86 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/d9dab9b1df1f45e2831d8ef068c62e86 2024-11-18T20:42:42,263 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/5809ea51939844d5a4a1858b572312b6 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/5809ea51939844d5a4a1858b572312b6 2024-11-18T20:42:42,263 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/9f31885e21e646728fca455f1292284a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/9f31885e21e646728fca455f1292284a 2024-11-18T20:42:42,264 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/ec528ecd9da34d70a9f90be6dd4591f1 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/ec528ecd9da34d70a9f90be6dd4591f1 2024-11-18T20:42:42,265 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/bc269f474fae4d9088c863ece66f1c49 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/bc269f474fae4d9088c863ece66f1c49 2024-11-18T20:42:42,266 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/6896a8b97a8c4042ab523a04140e4ca0 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/6896a8b97a8c4042ab523a04140e4ca0 2024-11-18T20:42:42,268 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/8aae9cc896c9464281a31489982c2ddc to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/8aae9cc896c9464281a31489982c2ddc 2024-11-18T20:42:42,269 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/eef0fa88f88d4b93ae145d070eb771c9 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/eef0fa88f88d4b93ae145d070eb771c9 2024-11-18T20:42:42,270 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/87e3fee24642415594829f4d2e41fce8 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/87e3fee24642415594829f4d2e41fce8 2024-11-18T20:42:42,272 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/d3d0597c149a478da9794511e8fee9ba, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/f55207773a2b458baac51b8eb2956c8b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/2bd69bbd5f6448088a7d883d7c5cd1e7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/3568da97debb4da2954bb0559c7ffc6c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/49b181f62d404adbbe248e89e5c20b5b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/e8d1d4c5cd7f4a69a48335772cb84198, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/8a15d7e691954aa684ac54802ccde0d4, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/505e2eb448694f9590a06a0d52483736, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/8747d700335748998febe0f539c23f5c] to archive 2024-11-18T20:42:42,274 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:42:42,277 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/d3d0597c149a478da9794511e8fee9ba to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/d3d0597c149a478da9794511e8fee9ba 2024-11-18T20:42:42,279 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/f55207773a2b458baac51b8eb2956c8b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/f55207773a2b458baac51b8eb2956c8b 2024-11-18T20:42:42,279 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/2bd69bbd5f6448088a7d883d7c5cd1e7 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/2bd69bbd5f6448088a7d883d7c5cd1e7 2024-11-18T20:42:42,281 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/3568da97debb4da2954bb0559c7ffc6c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/3568da97debb4da2954bb0559c7ffc6c 2024-11-18T20:42:42,282 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/49b181f62d404adbbe248e89e5c20b5b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/49b181f62d404adbbe248e89e5c20b5b 2024-11-18T20:42:42,285 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/e8d1d4c5cd7f4a69a48335772cb84198 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/e8d1d4c5cd7f4a69a48335772cb84198 2024-11-18T20:42:42,286 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/8a15d7e691954aa684ac54802ccde0d4 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/8a15d7e691954aa684ac54802ccde0d4 2024-11-18T20:42:42,287 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/505e2eb448694f9590a06a0d52483736 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/505e2eb448694f9590a06a0d52483736 2024-11-18T20:42:42,289 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/8747d700335748998febe0f539c23f5c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/8747d700335748998febe0f539c23f5c 2024-11-18T20:42:42,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:42,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962622317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:42,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:42,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962622317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:42,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962622537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:42,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962622537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:42,604 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/af388636adf64e24a63b1a759fd389eb 2024-11-18T20:42:42,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/9abbaaeb5947412eb8f8944a444b6aa3 is 50, key is test_row_0/B:col10/1731962559907/Put/seqid=0 2024-11-18T20:42:42,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742224_1400 (size=12151) 2024-11-18T20:42:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:42:42,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962622857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:42,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962622857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:43,015 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/9abbaaeb5947412eb8f8944a444b6aa3 2024-11-18T20:42:43,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/51c63d08faf44d6aa290a0de98555f2c is 50, key is test_row_0/C:col10/1731962559907/Put/seqid=0 2024-11-18T20:42:43,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742225_1401 (size=12151) 2024-11-18T20:42:43,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962623377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:43,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962623377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:43,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:43,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:43,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:43,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:43,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:43,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:43,425 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/51c63d08faf44d6aa290a0de98555f2c 2024-11-18T20:42:43,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/af388636adf64e24a63b1a759fd389eb as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/af388636adf64e24a63b1a759fd389eb 2024-11-18T20:42:43,436 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/af388636adf64e24a63b1a759fd389eb, entries=150, sequenceid=168, filesize=30.4 K 2024-11-18T20:42:43,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/9abbaaeb5947412eb8f8944a444b6aa3 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/9abbaaeb5947412eb8f8944a444b6aa3 2024-11-18T20:42:43,442 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/9abbaaeb5947412eb8f8944a444b6aa3, entries=150, sequenceid=168, filesize=11.9 K 2024-11-18T20:42:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/51c63d08faf44d6aa290a0de98555f2c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/51c63d08faf44d6aa290a0de98555f2c 2024-11-18T20:42:43,452 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/51c63d08faf44d6aa290a0de98555f2c, entries=150, sequenceid=168, filesize=11.9 K 2024-11-18T20:42:43,453 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 75114f181d047350ddda984d66f6e223 in 1670ms, sequenceid=168, compaction requested=true 2024-11-18T20:42:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-18T20:42:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-18T20:42:43,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-18T20:42:43,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8240 sec 2024-11-18T20:42:43,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.8280 sec 2024-11-18T20:42:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:42:43,777 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:43,777 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:42:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-18T20:42:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T20:42:43,779 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:42:43,780 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:42:43,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:42:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T20:42:43,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:42:43,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:43,933 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 75114f181d047350ddda984d66f6e223 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T20:42:43,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=A 2024-11-18T20:42:43,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:43,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=B 2024-11-18T20:42:43,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:43,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=C 2024-11-18T20:42:43,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:43,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b65101d2f5154c8886557fd1b1b9270a_75114f181d047350ddda984d66f6e223 is 50, key is test_row_0/A:col10/1731962562194/Put/seqid=0 2024-11-18T20:42:43,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742226_1402 (size=12304) 2024-11-18T20:42:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T20:42:44,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:44,350 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b65101d2f5154c8886557fd1b1b9270a_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b65101d2f5154c8886557fd1b1b9270a_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:44,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/1d8c4cf0c11e46948fc2f05fa72a22bf, store: [table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:44,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/1d8c4cf0c11e46948fc2f05fa72a22bf is 175, key is test_row_0/A:col10/1731962562194/Put/seqid=0 2024-11-18T20:42:44,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742227_1403 (size=31105) 2024-11-18T20:42:44,356 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/1d8c4cf0c11e46948fc2f05fa72a22bf 2024-11-18T20:42:44,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/9c2c83d8593949868ae3b778ef38e409 is 50, key is test_row_0/B:col10/1731962562194/Put/seqid=0 2024-11-18T20:42:44,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742228_1404 (size=12151) 2024-11-18T20:42:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:44,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T20:42:44,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962624452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:44,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962624453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:44,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962624577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:44,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962624577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:44,766 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/9c2c83d8593949868ae3b778ef38e409 2024-11-18T20:42:44,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/2df2fbebe3cc4a61b2d31c702a694bce is 50, key is test_row_0/C:col10/1731962562194/Put/seqid=0 2024-11-18T20:42:44,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742229_1405 (size=12151) 2024-11-18T20:42:44,778 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/2df2fbebe3cc4a61b2d31c702a694bce 2024-11-18T20:42:44,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/1d8c4cf0c11e46948fc2f05fa72a22bf as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/1d8c4cf0c11e46948fc2f05fa72a22bf 2024-11-18T20:42:44,786 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/1d8c4cf0c11e46948fc2f05fa72a22bf, entries=150, sequenceid=186, filesize=30.4 K 2024-11-18T20:42:44,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/9c2c83d8593949868ae3b778ef38e409 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/9c2c83d8593949868ae3b778ef38e409 2024-11-18T20:42:44,790 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/9c2c83d8593949868ae3b778ef38e409, entries=150, sequenceid=186, filesize=11.9 K 2024-11-18T20:42:44,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/2df2fbebe3cc4a61b2d31c702a694bce as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/2df2fbebe3cc4a61b2d31c702a694bce 2024-11-18T20:42:44,794 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/2df2fbebe3cc4a61b2d31c702a694bce, entries=150, sequenceid=186, filesize=11.9 K 2024-11-18T20:42:44,795 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 75114f181d047350ddda984d66f6e223 in 862ms, sequenceid=186, compaction requested=true 2024-11-18T20:42:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-18T20:42:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-18T20:42:44,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-18T20:42:44,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0160 sec 2024-11-18T20:42:44,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.0210 sec 2024-11-18T20:42:44,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:44,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 75114f181d047350ddda984d66f6e223 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T20:42:44,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=A 2024-11-18T20:42:44,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:44,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=B 2024-11-18T20:42:44,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:44,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=C 2024-11-18T20:42:44,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:44,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118436a862c64954921af896f8e028e09d2_75114f181d047350ddda984d66f6e223 is 50, key is test_row_0/A:col10/1731962564450/Put/seqid=0 2024-11-18T20:42:44,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742230_1406 (size=17284) 2024-11-18T20:42:44,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T20:42:44,917 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:44,917 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:42:44,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-18T20:42:44,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:42:44,919 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:42:44,920 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:42:44,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:42:44,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:44,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962624914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:44,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:44,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962624920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:45,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:42:45,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:45,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962625037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:45,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:45,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:45,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:45,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962625037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:45,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:45,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:45,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:45,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:45,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:45,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:45,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:45,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:45,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:45,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:45,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:45,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:45,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,232 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:45,236 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118436a862c64954921af896f8e028e09d2_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118436a862c64954921af896f8e028e09d2_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:45,236 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/4bc676c3012a48418d42feed3db9a324, store: [table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:45,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/4bc676c3012a48418d42feed3db9a324 is 175, key is test_row_0/A:col10/1731962564450/Put/seqid=0 2024-11-18T20:42:45,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:42:45,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742231_1407 (size=48389) 2024-11-18T20:42:45,243 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/4bc676c3012a48418d42feed3db9a324 2024-11-18T20:42:45,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/7d07784ddec34173a3da28f39674c290 is 50, key is test_row_0/B:col10/1731962564450/Put/seqid=0 2024-11-18T20:42:45,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742232_1408 (size=12151) 2024-11-18T20:42:45,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962625257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:45,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962625257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:45,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:45,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:45,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:45,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:45,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:45,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:45,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:45,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:45,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:42:45,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962625577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:45,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:45,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:45,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962625577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:45,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:45,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:45,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:45,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:45,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/7d07784ddec34173a3da28f39674c290 2024-11-18T20:42:45,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/c9068b8530784e56b938fee9bf9b273b is 50, key is test_row_0/C:col10/1731962564450/Put/seqid=0 2024-11-18T20:42:45,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742233_1409 (size=12151) 2024-11-18T20:42:45,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:45,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:45,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:45,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:45,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:45,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:45,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:45,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:45,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:45,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:45,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:45,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:45,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:42:46,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/c9068b8530784e56b938fee9bf9b273b 2024-11-18T20:42:46,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/4bc676c3012a48418d42feed3db9a324 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/4bc676c3012a48418d42feed3db9a324 2024-11-18T20:42:46,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/4bc676c3012a48418d42feed3db9a324, entries=250, sequenceid=202, filesize=47.3 K 2024-11-18T20:42:46,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/7d07784ddec34173a3da28f39674c290 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/7d07784ddec34173a3da28f39674c290 2024-11-18T20:42:46,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/7d07784ddec34173a3da28f39674c290, entries=150, sequenceid=202, filesize=11.9 K 2024-11-18T20:42:46,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/c9068b8530784e56b938fee9bf9b273b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/c9068b8530784e56b938fee9bf9b273b 2024-11-18T20:42:46,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/c9068b8530784e56b938fee9bf9b273b, entries=150, sequenceid=202, filesize=11.9 K 2024-11-18T20:42:46,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 75114f181d047350ddda984d66f6e223 in 1256ms, sequenceid=202, compaction requested=true 2024-11-18T20:42:46,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:46,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75114f181d047350ddda984d66f6e223:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:42:46,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:46,079 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-18T20:42:46,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75114f181d047350ddda984d66f6e223:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:42:46,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:46,079 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-18T20:42:46,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75114f181d047350ddda984d66f6e223:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:42:46,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:46,080 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 173117 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-18T20:42:46,080 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61063 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-18T20:42:46,080 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 75114f181d047350ddda984d66f6e223/B is initiating minor compaction (all files) 2024-11-18T20:42:46,080 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 75114f181d047350ddda984d66f6e223/A is initiating minor compaction (all files) 2024-11-18T20:42:46,080 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75114f181d047350ddda984d66f6e223/A in TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:46,080 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75114f181d047350ddda984d66f6e223/B in TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:46,080 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/f86dded809554e45adee1600175094a3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/7456076980cf4c67a985b22aa5bf6fe2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/9abbaaeb5947412eb8f8944a444b6aa3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/9c2c83d8593949868ae3b778ef38e409, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/7d07784ddec34173a3da28f39674c290] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp, totalSize=59.6 K 2024-11-18T20:42:46,080 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/7018b79192f84d83828f229186bfa5ec, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/678e6e94a5b84435b353de460c8a4998, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/af388636adf64e24a63b1a759fd389eb, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/1d8c4cf0c11e46948fc2f05fa72a22bf, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/4bc676c3012a48418d42feed3db9a324] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp, totalSize=169.1 K 2024-11-18T20:42:46,080 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:46,080 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/7018b79192f84d83828f229186bfa5ec, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/678e6e94a5b84435b353de460c8a4998, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/af388636adf64e24a63b1a759fd389eb, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/1d8c4cf0c11e46948fc2f05fa72a22bf, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/4bc676c3012a48418d42feed3db9a324] 2024-11-18T20:42:46,080 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting f86dded809554e45adee1600175094a3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731962558313 2024-11-18T20:42:46,081 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 7456076980cf4c67a985b22aa5bf6fe2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731962558692 2024-11-18T20:42:46,081 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7018b79192f84d83828f229186bfa5ec, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731962558313 2024-11-18T20:42:46,081 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 9abbaaeb5947412eb8f8944a444b6aa3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731962559889 2024-11-18T20:42:46,081 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 678e6e94a5b84435b353de460c8a4998, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731962558692 2024-11-18T20:42:46,081 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c2c83d8593949868ae3b778ef38e409, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731962562179 2024-11-18T20:42:46,081 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting af388636adf64e24a63b1a759fd389eb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731962559889 2024-11-18T20:42:46,081 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d07784ddec34173a3da28f39674c290, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731962564436 2024-11-18T20:42:46,082 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d8c4cf0c11e46948fc2f05fa72a22bf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731962562179 2024-11-18T20:42:46,082 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bc676c3012a48418d42feed3db9a324, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731962564436 2024-11-18T20:42:46,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 75114f181d047350ddda984d66f6e223 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T20:42:46,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=A 2024-11-18T20:42:46,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:46,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=B 2024-11-18T20:42:46,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:46,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=C 2024-11-18T20:42:46,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:46,105 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75114f181d047350ddda984d66f6e223#B#compaction#347 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:46,105 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/ba60941f744a4011a4550bf3170c6b3c is 50, key is test_row_0/B:col10/1731962564450/Put/seqid=0 2024-11-18T20:42:46,110 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:46,112 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118fd8be18410664f5294ff351d9462050f_75114f181d047350ddda984d66f6e223 store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:46,116 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118fd8be18410664f5294ff351d9462050f_75114f181d047350ddda984d66f6e223, store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:46,116 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118fd8be18410664f5294ff351d9462050f_75114f181d047350ddda984d66f6e223 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:46,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742234_1410 (size=12323) 2024-11-18T20:42:46,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184b08380ea3ba43a9859075d6b8b19a85_75114f181d047350ddda984d66f6e223 is 50, key is test_row_0/A:col10/1731962564899/Put/seqid=0 2024-11-18T20:42:46,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742235_1411 (size=4469) 2024-11-18T20:42:46,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742236_1412 (size=14794) 2024-11-18T20:42:46,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:46,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:46,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:46,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:46,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962626174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:46,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:46,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:46,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:46,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962626184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:46,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:46,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:46,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:46,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:46,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:46,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:46,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:46,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962626300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:46,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962626301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:46,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:46,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:46,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:46,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:46,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,524 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/ba60941f744a4011a4550bf3170c6b3c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/ba60941f744a4011a4550bf3170c6b3c 2024-11-18T20:42:46,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962626517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:46,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962626517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:46,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:46,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:46,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:46,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:46,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:46,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:46,532 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 75114f181d047350ddda984d66f6e223/B of 75114f181d047350ddda984d66f6e223 into ba60941f744a4011a4550bf3170c6b3c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:46,532 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:46,532 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., storeName=75114f181d047350ddda984d66f6e223/B, priority=11, startTime=1731962566079; duration=0sec 2024-11-18T20:42:46,532 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:46,532 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75114f181d047350ddda984d66f6e223:B 2024-11-18T20:42:46,532 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-18T20:42:46,533 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61063 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-18T20:42:46,533 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 75114f181d047350ddda984d66f6e223/C is initiating minor compaction (all files) 2024-11-18T20:42:46,533 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75114f181d047350ddda984d66f6e223/C in TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:46,533 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/9fa384e089dc4cb98293b0af929e5fc0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/1f2309c80c4141acaefb1a2140fc3f01, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/51c63d08faf44d6aa290a0de98555f2c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/2df2fbebe3cc4a61b2d31c702a694bce, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/c9068b8530784e56b938fee9bf9b273b] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp, totalSize=59.6 K 2024-11-18T20:42:46,533 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fa384e089dc4cb98293b0af929e5fc0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731962558313 2024-11-18T20:42:46,533 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f2309c80c4141acaefb1a2140fc3f01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731962558692 2024-11-18T20:42:46,533 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 51c63d08faf44d6aa290a0de98555f2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731962559889 2024-11-18T20:42:46,534 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 2df2fbebe3cc4a61b2d31c702a694bce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731962562179 2024-11-18T20:42:46,534 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting c9068b8530784e56b938fee9bf9b273b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731962564436 2024-11-18T20:42:46,535 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:46,535 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75114f181d047350ddda984d66f6e223#A#compaction#348 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:46,535 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/79538d4fffc946bbabab5461f84c9db4 is 175, key is test_row_0/A:col10/1731962564450/Put/seqid=0 2024-11-18T20:42:46,538 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184b08380ea3ba43a9859075d6b8b19a85_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184b08380ea3ba43a9859075d6b8b19a85_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:46,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742237_1413 (size=31277) 2024-11-18T20:42:46,540 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/c135ab0ddf8d444489aa8e884d423d11, store: [table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:46,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/c135ab0ddf8d444489aa8e884d423d11 is 175, key is test_row_0/A:col10/1731962564899/Put/seqid=0 2024-11-18T20:42:46,545 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75114f181d047350ddda984d66f6e223#C#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:46,546 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/51a30f5777ea4a898c68400c76059d08 is 50, key is test_row_0/C:col10/1731962564450/Put/seqid=0 2024-11-18T20:42:46,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742238_1414 (size=39749) 2024-11-18T20:42:46,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742239_1415 (size=12323) 2024-11-18T20:42:46,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:46,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:46,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:46,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:46,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:46,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:46,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:46,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:46,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962626837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:46,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:46,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:46,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:46,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962626837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:46,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:46,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:46,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:46,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:46,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:46,942 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/79538d4fffc946bbabab5461f84c9db4 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/79538d4fffc946bbabab5461f84c9db4 2024-11-18T20:42:46,946 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 75114f181d047350ddda984d66f6e223/A of 75114f181d047350ddda984d66f6e223 into 79538d4fffc946bbabab5461f84c9db4(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:46,946 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:46,946 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., storeName=75114f181d047350ddda984d66f6e223/A, priority=11, startTime=1731962566078; duration=0sec 2024-11-18T20:42:46,946 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:46,947 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75114f181d047350ddda984d66f6e223:A 2024-11-18T20:42:46,948 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/c135ab0ddf8d444489aa8e884d423d11 2024-11-18T20:42:46,953 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/51a30f5777ea4a898c68400c76059d08 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/51a30f5777ea4a898c68400c76059d08 2024-11-18T20:42:46,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/6d78a0dd43034ac489789b1bb2ac1809 is 50, key is test_row_0/B:col10/1731962564899/Put/seqid=0 2024-11-18T20:42:46,959 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 75114f181d047350ddda984d66f6e223/C of 75114f181d047350ddda984d66f6e223 into 51a30f5777ea4a898c68400c76059d08(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:46,959 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:46,959 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., storeName=75114f181d047350ddda984d66f6e223/C, priority=11, startTime=1731962566079; duration=0sec 2024-11-18T20:42:46,959 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:46,959 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75114f181d047350ddda984d66f6e223:C 2024-11-18T20:42:46,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742240_1416 (size=12151) 2024-11-18T20:42:47,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:47,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:47,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:47,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:47,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:42:47,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:47,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:47,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:47,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:47,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962627347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:47,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962627348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:47,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/6d78a0dd43034ac489789b1bb2ac1809 2024-11-18T20:42:47,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:47,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:47,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:47,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:47,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/5cbaea21ce3047deb0b4d9c856e53972 is 50, key is test_row_0/C:col10/1731962564899/Put/seqid=0 2024-11-18T20:42:47,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742241_1417 (size=12151) 2024-11-18T20:42:47,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/5cbaea21ce3047deb0b4d9c856e53972 2024-11-18T20:42:47,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/c135ab0ddf8d444489aa8e884d423d11 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/c135ab0ddf8d444489aa8e884d423d11 2024-11-18T20:42:47,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/c135ab0ddf8d444489aa8e884d423d11, entries=200, sequenceid=219, filesize=38.8 K 2024-11-18T20:42:47,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/6d78a0dd43034ac489789b1bb2ac1809 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/6d78a0dd43034ac489789b1bb2ac1809 2024-11-18T20:42:47,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/6d78a0dd43034ac489789b1bb2ac1809, entries=150, sequenceid=219, filesize=11.9 K 2024-11-18T20:42:47,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/5cbaea21ce3047deb0b4d9c856e53972 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/5cbaea21ce3047deb0b4d9c856e53972 2024-11-18T20:42:47,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/5cbaea21ce3047deb0b4d9c856e53972, entries=150, sequenceid=219, filesize=11.9 K 2024-11-18T20:42:47,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 75114f181d047350ddda984d66f6e223 in 1294ms, sequenceid=219, compaction requested=false 2024-11-18T20:42:47,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:47,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:47,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 75114f181d047350ddda984d66f6e223 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T20:42:47,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=A 2024-11-18T20:42:47,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:47,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=B 2024-11-18T20:42:47,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:47,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=C 2024-11-18T20:42:47,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:47,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111843292ca8beb646f7ad21889e02583c14_75114f181d047350ddda984d66f6e223 is 50, key is test_row_0/A:col10/1731962566182/Put/seqid=0 2024-11-18T20:42:47,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742242_1418 (size=14794) 2024-11-18T20:42:47,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43882 deadline: 1731962627478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:47,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1731962627482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:47,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43860 deadline: 1731962627487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:47,497 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in 75114f181d047350ddda984d66f6e223 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18376 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:42:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:47,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:47,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:47,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:47,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:47,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43882 deadline: 1731962627597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:47,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:47,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:47,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:47,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1731962627607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:47,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:47,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:47,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:47,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:47,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:47,822 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:47,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:47,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:47,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:47,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43882 deadline: 1731962627817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:47,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1731962627817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:47,826 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111843292ca8beb646f7ad21889e02583c14_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111843292ca8beb646f7ad21889e02583c14_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:47,827 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/450a00ef699143348aa8fcf95a70c845, store: [table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:47,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/450a00ef699143348aa8fcf95a70c845 is 175, key is test_row_0/A:col10/1731962566182/Put/seqid=0 2024-11-18T20:42:47,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742243_1419 (size=39749) 2024-11-18T20:42:47,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:47,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:47,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:47,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:47,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:48,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:48,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:48,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:48,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:48,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:48,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:48,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43882 deadline: 1731962628127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:48,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1731962628127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:48,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:48,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:48,231 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/450a00ef699143348aa8fcf95a70c845 2024-11-18T20:42:48,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/531e37fe82884c95887370608124e100 is 50, key is test_row_0/B:col10/1731962566182/Put/seqid=0 2024-11-18T20:42:48,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742244_1420 (size=12151) 2024-11-18T20:42:48,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:48,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:48,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:48,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:48,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:48,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:48,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43894 deadline: 1731962628357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:48,367 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in 75114f181d047350ddda984d66f6e223 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:42:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:48,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:48,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43850 deadline: 1731962628367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:48,371 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in 75114f181d047350ddda984d66f6e223 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:42:48,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:48,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:48,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:48,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:48,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:48,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:48,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:48,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:48,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:48,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:48,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:48,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:48,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:48,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:48,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:48,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/531e37fe82884c95887370608124e100 2024-11-18T20:42:48,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/84f09b700ce34e768657ef5878a3f070 is 50, key is test_row_0/C:col10/1731962566182/Put/seqid=0 2024-11-18T20:42:48,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742245_1421 (size=12151) 2024-11-18T20:42:48,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43882 deadline: 1731962628647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:48,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:42:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43840 deadline: 1731962628647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:42:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:42:48,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:48,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:48,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:48,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:48,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:48,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:48,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:48,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:49,014 INFO [Thread-1623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1623. 2024-11-18T20:42:49,014 DEBUG [Thread-1623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:49,014 DEBUG [Thread-1623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:49,014 DEBUG [Thread-1623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:49,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:49,021 INFO [Thread-1617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1617. 2024-11-18T20:42:49,021 DEBUG [Thread-1617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:49,021 DEBUG [Thread-1617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:49,021 DEBUG [Thread-1617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:49,021 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:49,026 INFO [Thread-1621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1621. 2024-11-18T20:42:49,026 DEBUG [Thread-1621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:49,026 DEBUG [Thread-1621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:49,026 DEBUG [Thread-1621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:49,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:49,031 INFO [Thread-1625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1625. 2024-11-18T20:42:49,031 DEBUG [Thread-1625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:49,031 DEBUG [Thread-1625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:49,032 DEBUG [Thread-1625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:49,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:49,033 INFO [Thread-1619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1619. 2024-11-18T20:42:49,033 DEBUG [Thread-1619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:49,033 DEBUG [Thread-1619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:49,033 DEBUG [Thread-1619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:49,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:49,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:49,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:49,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:49,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:49,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:49,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:42:49,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/84f09b700ce34e768657ef5878a3f070 2024-11-18T20:42:49,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/450a00ef699143348aa8fcf95a70c845 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/450a00ef699143348aa8fcf95a70c845 2024-11-18T20:42:49,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/450a00ef699143348aa8fcf95a70c845, entries=200, sequenceid=238, filesize=38.8 K 2024-11-18T20:42:49,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/531e37fe82884c95887370608124e100 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/531e37fe82884c95887370608124e100 2024-11-18T20:42:49,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/531e37fe82884c95887370608124e100, entries=150, sequenceid=238, filesize=11.9 K 2024-11-18T20:42:49,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/84f09b700ce34e768657ef5878a3f070 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/84f09b700ce34e768657ef5878a3f070 2024-11-18T20:42:49,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/84f09b700ce34e768657ef5878a3f070, entries=150, sequenceid=238, filesize=11.9 K 2024-11-18T20:42:49,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 75114f181d047350ddda984d66f6e223 in 1662ms, sequenceid=238, compaction requested=true 2024-11-18T20:42:49,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:49,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75114f181d047350ddda984d66f6e223:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:42:49,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:49,075 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:49,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75114f181d047350ddda984d66f6e223:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:42:49,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:49,075 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:49,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 75114f181d047350ddda984d66f6e223:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:42:49,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:49,076 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36625 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:49,076 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110775 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:49,076 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 75114f181d047350ddda984d66f6e223/B is initiating minor compaction (all files) 2024-11-18T20:42:49,076 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 75114f181d047350ddda984d66f6e223/A is initiating minor compaction (all files) 2024-11-18T20:42:49,076 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75114f181d047350ddda984d66f6e223/B in TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:49,076 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75114f181d047350ddda984d66f6e223/A in TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:49,076 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/ba60941f744a4011a4550bf3170c6b3c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/6d78a0dd43034ac489789b1bb2ac1809, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/531e37fe82884c95887370608124e100] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp, totalSize=35.8 K 2024-11-18T20:42:49,076 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/79538d4fffc946bbabab5461f84c9db4, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/c135ab0ddf8d444489aa8e884d423d11, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/450a00ef699143348aa8fcf95a70c845] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp, totalSize=108.2 K 2024-11-18T20:42:49,076 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:49,076 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/79538d4fffc946bbabab5461f84c9db4, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/c135ab0ddf8d444489aa8e884d423d11, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/450a00ef699143348aa8fcf95a70c845] 2024-11-18T20:42:49,076 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting ba60941f744a4011a4550bf3170c6b3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731962564436 2024-11-18T20:42:49,076 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79538d4fffc946bbabab5461f84c9db4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731962564436 2024-11-18T20:42:49,076 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d78a0dd43034ac489789b1bb2ac1809, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731962564881 2024-11-18T20:42:49,076 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting c135ab0ddf8d444489aa8e884d423d11, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731962564881 2024-11-18T20:42:49,077 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 450a00ef699143348aa8fcf95a70c845, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731962566146 2024-11-18T20:42:49,077 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 531e37fe82884c95887370608124e100, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731962566153 2024-11-18T20:42:49,082 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:49,083 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75114f181d047350ddda984d66f6e223#B#compaction#356 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:49,083 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411186592c7eb0b1f4c7d8dd19783ffede3b0_75114f181d047350ddda984d66f6e223 store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:49,083 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/d521785dc6fb4226b7cb48430a6e17f3 is 50, key is test_row_0/B:col10/1731962566182/Put/seqid=0 2024-11-18T20:42:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:42:49,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742246_1422 (size=12425) 2024-11-18T20:42:49,094 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411186592c7eb0b1f4c7d8dd19783ffede3b0_75114f181d047350ddda984d66f6e223, store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:49,094 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186592c7eb0b1f4c7d8dd19783ffede3b0_75114f181d047350ddda984d66f6e223 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:49,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742247_1423 (size=4469) 2024-11-18T20:42:49,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:42:49,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:49,199 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 75114f181d047350ddda984d66f6e223 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T20:42:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=A 2024-11-18T20:42:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=B 2024-11-18T20:42:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=C 2024-11-18T20:42:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:49,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c36610bac6fa4cdb95f2d84cf5aa5ca1_75114f181d047350ddda984d66f6e223 is 50, key is test_row_0/A:col10/1731962567477/Put/seqid=0 2024-11-18T20:42:49,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742248_1424 (size=12304) 2024-11-18T20:42:49,495 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/d521785dc6fb4226b7cb48430a6e17f3 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/d521785dc6fb4226b7cb48430a6e17f3 2024-11-18T20:42:49,500 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75114f181d047350ddda984d66f6e223#A#compaction#357 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:49,501 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/a10e344e29684d1487633ef2f6e0c6eb is 175, key is test_row_0/A:col10/1731962566182/Put/seqid=0 2024-11-18T20:42:49,502 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75114f181d047350ddda984d66f6e223/B of 75114f181d047350ddda984d66f6e223 into d521785dc6fb4226b7cb48430a6e17f3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:49,502 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:49,502 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., storeName=75114f181d047350ddda984d66f6e223/B, priority=13, startTime=1731962569075; duration=0sec 2024-11-18T20:42:49,503 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:42:49,503 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75114f181d047350ddda984d66f6e223:B 2024-11-18T20:42:49,503 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:42:49,504 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36625 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:42:49,504 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 75114f181d047350ddda984d66f6e223/C is initiating minor compaction (all files) 2024-11-18T20:42:49,504 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 75114f181d047350ddda984d66f6e223/C in TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:49,505 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/51a30f5777ea4a898c68400c76059d08, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/5cbaea21ce3047deb0b4d9c856e53972, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/84f09b700ce34e768657ef5878a3f070] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp, totalSize=35.8 K 2024-11-18T20:42:49,505 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 51a30f5777ea4a898c68400c76059d08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731962564436 2024-11-18T20:42:49,505 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cbaea21ce3047deb0b4d9c856e53972, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731962564881 2024-11-18T20:42:49,506 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 84f09b700ce34e768657ef5878a3f070, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731962566153 2024-11-18T20:42:49,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742249_1425 (size=31379) 2024-11-18T20:42:49,513 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 75114f181d047350ddda984d66f6e223#C#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:42:49,514 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/a3099ccbeb454d7d99e80b871db77d41 is 50, key is test_row_0/C:col10/1731962566182/Put/seqid=0 2024-11-18T20:42:49,514 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/a10e344e29684d1487633ef2f6e0c6eb as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/a10e344e29684d1487633ef2f6e0c6eb 2024-11-18T20:42:49,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742250_1426 (size=12425) 2024-11-18T20:42:49,519 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75114f181d047350ddda984d66f6e223/A of 75114f181d047350ddda984d66f6e223 into a10e344e29684d1487633ef2f6e0c6eb(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:49,519 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:49,520 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., storeName=75114f181d047350ddda984d66f6e223/A, priority=13, startTime=1731962569075; duration=0sec 2024-11-18T20:42:49,520 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:49,520 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75114f181d047350ddda984d66f6e223:A 2024-11-18T20:42:49,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:49,614 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c36610bac6fa4cdb95f2d84cf5aa5ca1_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c36610bac6fa4cdb95f2d84cf5aa5ca1_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:49,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/d2131f09a9604d15b25d1f3b97c93b77, store: [table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:49,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/d2131f09a9604d15b25d1f3b97c93b77 is 175, key is test_row_0/A:col10/1731962567477/Put/seqid=0 2024-11-18T20:42:49,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742251_1427 (size=31105) 2024-11-18T20:42:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:49,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. as already flushing 2024-11-18T20:42:49,669 INFO [Thread-1614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1614. 2024-11-18T20:42:49,669 DEBUG [Thread-1614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:49,669 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:49,669 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:49,669 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:49,679 INFO [Thread-1610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1610. 2024-11-18T20:42:49,679 DEBUG [Thread-1610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:49,679 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:49,679 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:49,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:49,921 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/a3099ccbeb454d7d99e80b871db77d41 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/a3099ccbeb454d7d99e80b871db77d41 2024-11-18T20:42:49,925 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 75114f181d047350ddda984d66f6e223/C of 75114f181d047350ddda984d66f6e223 into a3099ccbeb454d7d99e80b871db77d41(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:42:49,925 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:49,925 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., storeName=75114f181d047350ddda984d66f6e223/C, priority=13, startTime=1731962569075; duration=0sec 2024-11-18T20:42:49,925 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:42:49,925 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 75114f181d047350ddda984d66f6e223:C 2024-11-18T20:42:50,019 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/d2131f09a9604d15b25d1f3b97c93b77 2024-11-18T20:42:50,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/fb78533f70f44611b860f49baf7a37bd is 50, key is test_row_0/B:col10/1731962567477/Put/seqid=0 2024-11-18T20:42:50,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742252_1428 (size=12151) 2024-11-18T20:42:50,033 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/fb78533f70f44611b860f49baf7a37bd 2024-11-18T20:42:50,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/36415b960f2748f295cf30e2a1d138f1 is 50, key is test_row_0/C:col10/1731962567477/Put/seqid=0 2024-11-18T20:42:50,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742253_1429 (size=12151) 2024-11-18T20:42:50,391 INFO [Thread-1612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1612. 2024-11-18T20:42:50,391 DEBUG [Thread-1612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:50,391 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:50,391 INFO [Thread-1608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1608. 2024-11-18T20:42:50,391 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:50,391 DEBUG [Thread-1608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:50,391 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:50,391 DEBUG [Thread-1608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:50,392 DEBUG [Thread-1608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:50,392 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:50,443 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/36415b960f2748f295cf30e2a1d138f1 2024-11-18T20:42:50,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/d2131f09a9604d15b25d1f3b97c93b77 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/d2131f09a9604d15b25d1f3b97c93b77 2024-11-18T20:42:50,449 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/d2131f09a9604d15b25d1f3b97c93b77, entries=150, sequenceid=254, filesize=30.4 K 2024-11-18T20:42:50,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/fb78533f70f44611b860f49baf7a37bd as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/fb78533f70f44611b860f49baf7a37bd 2024-11-18T20:42:50,453 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/fb78533f70f44611b860f49baf7a37bd, entries=150, sequenceid=254, filesize=11.9 K 2024-11-18T20:42:50,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/36415b960f2748f295cf30e2a1d138f1 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/36415b960f2748f295cf30e2a1d138f1 2024-11-18T20:42:50,459 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/36415b960f2748f295cf30e2a1d138f1, entries=150, sequenceid=254, filesize=11.9 K 2024-11-18T20:42:50,460 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=26.84 KB/27480 for 75114f181d047350ddda984d66f6e223 in 1261ms, sequenceid=254, compaction requested=false 2024-11-18T20:42:50,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 75114f181d047350ddda984d66f6e223: 2024-11-18T20:42:50,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:50,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-18T20:42:50,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-18T20:42:50,464 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-18T20:42:50,464 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.5410 sec 2024-11-18T20:42:50,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 5.5480 sec 2024-11-18T20:42:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:42:53,098 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:57,572 INFO [Thread-1606 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1606. 2024-11-18T20:42:57,572 DEBUG [Thread-1606 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:42:57,572 DEBUG [Thread-1606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:57,572 DEBUG [Thread-1606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:57,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:57,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T20:42:57,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-18T20:42:57,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-18T20:42:57,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-18T20:42:57,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-18T20:42:57,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-18T20:42:57,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T20:42:57,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T20:42:57,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1077 2024-11-18T20:42:57,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3231 rows 2024-11-18T20:42:57,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1083 2024-11-18T20:42:57,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3237 rows 2024-11-18T20:42:57,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1084 2024-11-18T20:42:57,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3240 rows 2024-11-18T20:42:57,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1083 2024-11-18T20:42:57,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3245 rows 2024-11-18T20:42:57,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1079 2024-11-18T20:42:57,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3227 rows 2024-11-18T20:42:57,574 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:42:57,574 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:42:57,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:57,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:42:57,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:42:57,576 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T20:42:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T20:42:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T20:42:57,581 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962577580"}]},"ts":"1731962577580"} 2024-11-18T20:42:57,583 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T20:42:57,583 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T20:42:57,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:42:57,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=75114f181d047350ddda984d66f6e223, UNASSIGN}] 2024-11-18T20:42:57,587 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=75114f181d047350ddda984d66f6e223, UNASSIGN 2024-11-18T20:42:57,588 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=75114f181d047350ddda984d66f6e223, regionState=CLOSING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:42:57,590 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=75114f181d047350ddda984d66f6e223, UNASSIGN because future has completed 2024-11-18T20:42:57,590 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:42:57,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:42:57,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T20:42:57,743 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:57,743 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:42:57,743 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing 75114f181d047350ddda984d66f6e223, disabling compactions & flushes 2024-11-18T20:42:57,743 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:57,743 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:57,743 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. after waiting 0 ms 2024-11-18T20:42:57,743 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:57,743 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing 75114f181d047350ddda984d66f6e223 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-18T20:42:57,743 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=A 2024-11-18T20:42:57,743 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:57,743 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=B 2024-11-18T20:42:57,743 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:57,743 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 75114f181d047350ddda984d66f6e223, store=C 2024-11-18T20:42:57,743 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:42:57,749 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d6528e73feb64294a688186e886c4bea_75114f181d047350ddda984d66f6e223 is 50, key is test_row_0/A:col10/1731962570388/Put/seqid=0 2024-11-18T20:42:57,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742254_1430 (size=12454) 2024-11-18T20:42:57,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T20:42:58,154 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:58,158 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d6528e73feb64294a688186e886c4bea_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d6528e73feb64294a688186e886c4bea_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:58,159 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/39148337aa4f4f07ae5e411f22b85333, store: [table=TestAcidGuarantees family=A region=75114f181d047350ddda984d66f6e223] 2024-11-18T20:42:58,160 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/39148337aa4f4f07ae5e411f22b85333 is 175, key is test_row_0/A:col10/1731962570388/Put/seqid=0 2024-11-18T20:42:58,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742255_1431 (size=31255) 2024-11-18T20:42:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T20:42:58,564 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/39148337aa4f4f07ae5e411f22b85333 2024-11-18T20:42:58,572 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/bb423c5b8971420681277365facb63c7 is 50, key is test_row_0/B:col10/1731962570388/Put/seqid=0 2024-11-18T20:42:58,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742256_1432 (size=12301) 2024-11-18T20:42:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T20:42:58,976 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/bb423c5b8971420681277365facb63c7 2024-11-18T20:42:58,984 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/4fcaaec6d9ef4073b0b5543961486b9e is 50, key is test_row_0/C:col10/1731962570388/Put/seqid=0 2024-11-18T20:42:58,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742257_1433 (size=12301) 2024-11-18T20:42:59,388 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/4fcaaec6d9ef4073b0b5543961486b9e 2024-11-18T20:42:59,393 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/A/39148337aa4f4f07ae5e411f22b85333 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/39148337aa4f4f07ae5e411f22b85333 2024-11-18T20:42:59,397 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/39148337aa4f4f07ae5e411f22b85333, entries=150, sequenceid=265, filesize=30.5 K 2024-11-18T20:42:59,399 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/B/bb423c5b8971420681277365facb63c7 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/bb423c5b8971420681277365facb63c7 2024-11-18T20:42:59,402 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/bb423c5b8971420681277365facb63c7, entries=150, sequenceid=265, filesize=12.0 K 2024-11-18T20:42:59,403 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/.tmp/C/4fcaaec6d9ef4073b0b5543961486b9e as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/4fcaaec6d9ef4073b0b5543961486b9e 2024-11-18T20:42:59,407 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/4fcaaec6d9ef4073b0b5543961486b9e, entries=150, sequenceid=265, filesize=12.0 K 2024-11-18T20:42:59,409 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 75114f181d047350ddda984d66f6e223 in 1665ms, sequenceid=265, compaction requested=true 2024-11-18T20:42:59,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/7018b79192f84d83828f229186bfa5ec, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/678e6e94a5b84435b353de460c8a4998, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/af388636adf64e24a63b1a759fd389eb, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/1d8c4cf0c11e46948fc2f05fa72a22bf, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/4bc676c3012a48418d42feed3db9a324, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/79538d4fffc946bbabab5461f84c9db4, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/c135ab0ddf8d444489aa8e884d423d11, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/450a00ef699143348aa8fcf95a70c845] to archive 2024-11-18T20:42:59,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:42:59,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/7018b79192f84d83828f229186bfa5ec to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/7018b79192f84d83828f229186bfa5ec 2024-11-18T20:42:59,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/678e6e94a5b84435b353de460c8a4998 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/678e6e94a5b84435b353de460c8a4998 2024-11-18T20:42:59,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/af388636adf64e24a63b1a759fd389eb to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/af388636adf64e24a63b1a759fd389eb 2024-11-18T20:42:59,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/1d8c4cf0c11e46948fc2f05fa72a22bf to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/1d8c4cf0c11e46948fc2f05fa72a22bf 2024-11-18T20:42:59,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/4bc676c3012a48418d42feed3db9a324 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/4bc676c3012a48418d42feed3db9a324 2024-11-18T20:42:59,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/79538d4fffc946bbabab5461f84c9db4 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/79538d4fffc946bbabab5461f84c9db4 2024-11-18T20:42:59,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/c135ab0ddf8d444489aa8e884d423d11 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/c135ab0ddf8d444489aa8e884d423d11 2024-11-18T20:42:59,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/450a00ef699143348aa8fcf95a70c845 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/450a00ef699143348aa8fcf95a70c845 2024-11-18T20:42:59,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/f86dded809554e45adee1600175094a3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/7456076980cf4c67a985b22aa5bf6fe2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/9abbaaeb5947412eb8f8944a444b6aa3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/9c2c83d8593949868ae3b778ef38e409, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/ba60941f744a4011a4550bf3170c6b3c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/7d07784ddec34173a3da28f39674c290, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/6d78a0dd43034ac489789b1bb2ac1809, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/531e37fe82884c95887370608124e100] to archive 2024-11-18T20:42:59,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:42:59,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/f86dded809554e45adee1600175094a3 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/f86dded809554e45adee1600175094a3 2024-11-18T20:42:59,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/7456076980cf4c67a985b22aa5bf6fe2 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/7456076980cf4c67a985b22aa5bf6fe2 2024-11-18T20:42:59,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/9abbaaeb5947412eb8f8944a444b6aa3 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/9abbaaeb5947412eb8f8944a444b6aa3 2024-11-18T20:42:59,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/9c2c83d8593949868ae3b778ef38e409 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/9c2c83d8593949868ae3b778ef38e409 2024-11-18T20:42:59,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/ba60941f744a4011a4550bf3170c6b3c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/ba60941f744a4011a4550bf3170c6b3c 2024-11-18T20:42:59,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/7d07784ddec34173a3da28f39674c290 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/7d07784ddec34173a3da28f39674c290 2024-11-18T20:42:59,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/6d78a0dd43034ac489789b1bb2ac1809 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/6d78a0dd43034ac489789b1bb2ac1809 2024-11-18T20:42:59,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/531e37fe82884c95887370608124e100 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/531e37fe82884c95887370608124e100 2024-11-18T20:42:59,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/9fa384e089dc4cb98293b0af929e5fc0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/1f2309c80c4141acaefb1a2140fc3f01, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/51c63d08faf44d6aa290a0de98555f2c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/2df2fbebe3cc4a61b2d31c702a694bce, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/51a30f5777ea4a898c68400c76059d08, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/c9068b8530784e56b938fee9bf9b273b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/5cbaea21ce3047deb0b4d9c856e53972, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/84f09b700ce34e768657ef5878a3f070] to archive 2024-11-18T20:42:59,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:42:59,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/9fa384e089dc4cb98293b0af929e5fc0 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/9fa384e089dc4cb98293b0af929e5fc0 2024-11-18T20:42:59,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/1f2309c80c4141acaefb1a2140fc3f01 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/1f2309c80c4141acaefb1a2140fc3f01 2024-11-18T20:42:59,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/51c63d08faf44d6aa290a0de98555f2c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/51c63d08faf44d6aa290a0de98555f2c 2024-11-18T20:42:59,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/2df2fbebe3cc4a61b2d31c702a694bce to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/2df2fbebe3cc4a61b2d31c702a694bce 2024-11-18T20:42:59,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/51a30f5777ea4a898c68400c76059d08 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/51a30f5777ea4a898c68400c76059d08 2024-11-18T20:42:59,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/c9068b8530784e56b938fee9bf9b273b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/c9068b8530784e56b938fee9bf9b273b 2024-11-18T20:42:59,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/5cbaea21ce3047deb0b4d9c856e53972 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/5cbaea21ce3047deb0b4d9c856e53972 2024-11-18T20:42:59,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/84f09b700ce34e768657ef5878a3f070 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/84f09b700ce34e768657ef5878a3f070 2024-11-18T20:42:59,448 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/recovered.edits/268.seqid, newMaxSeqId=268, maxSeqId=4 2024-11-18T20:42:59,448 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. 2024-11-18T20:42:59,448 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for 75114f181d047350ddda984d66f6e223: Waiting for close lock at 1731962577743Running coprocessor pre-close hooks at 1731962577743Disabling compacts and flushes for region at 1731962577743Disabling writes for close at 1731962577743Obtaining lock to block concurrent updates at 1731962577743Preparing flush snapshotting stores in 75114f181d047350ddda984d66f6e223 at 1731962577743Finished memstore snapshotting TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731962577744 (+1 ms)Flushing stores of TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223. at 1731962577744Flushing 75114f181d047350ddda984d66f6e223/A: creating writer at 1731962577744Flushing mob file 75114f181d047350ddda984d66f6e223/A: appending metadata at 1731962577749 (+5 ms)Flushing mob file 75114f181d047350ddda984d66f6e223/A: closing flushed file at 1731962577749Flushing 75114f181d047350ddda984d66f6e223/A: appending metadata at 1731962578159 (+410 ms)Flushing 75114f181d047350ddda984d66f6e223/A: closing flushed file at 1731962578159Flushing 75114f181d047350ddda984d66f6e223/B: creating writer at 1731962578569 (+410 ms)Flushing 75114f181d047350ddda984d66f6e223/B: appending metadata at 1731962578571 (+2 ms)Flushing 75114f181d047350ddda984d66f6e223/B: closing flushed file at 1731962578571Flushing 75114f181d047350ddda984d66f6e223/C: creating writer at 1731962578980 (+409 ms)Flushing 75114f181d047350ddda984d66f6e223/C: appending metadata at 1731962578983 (+3 ms)Flushing 75114f181d047350ddda984d66f6e223/C: closing flushed file at 1731962578983Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3dc3765f: reopening flushed file at 1731962579392 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@797b4936: reopening flushed file at 1731962579397 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26885135: reopening flushed file at 1731962579402 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 75114f181d047350ddda984d66f6e223 in 1665ms, sequenceid=265, compaction requested=true at 1731962579409 (+7 ms)Writing region close event to WAL at 1731962579445 (+36 ms)Running coprocessor post-close hooks at 1731962579448 (+3 ms)Closed at 1731962579448 2024-11-18T20:42:59,450 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed 75114f181d047350ddda984d66f6e223 2024-11-18T20:42:59,451 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=75114f181d047350ddda984d66f6e223, regionState=CLOSED 2024-11-18T20:42:59,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 because future has completed 2024-11-18T20:42:59,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-18T20:42:59,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure 75114f181d047350ddda984d66f6e223, server=c0a89b2656d4,45393,1731962440107 in 1.8630 sec 2024-11-18T20:42:59,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-18T20:42:59,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=75114f181d047350ddda984d66f6e223, UNASSIGN in 1.8710 sec 2024-11-18T20:42:59,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-18T20:42:59,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8760 sec 2024-11-18T20:42:59,464 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962579463"}]},"ts":"1731962579463"} 2024-11-18T20:42:59,465 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T20:42:59,465 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T20:42:59,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8900 sec 2024-11-18T20:42:59,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T20:42:59,728 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:59,729 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T20:42:59,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:42:59,732 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:42:59,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T20:42:59,733 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:42:59,737 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223 2024-11-18T20:42:59,740 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A, FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B, FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C, FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/recovered.edits] 2024-11-18T20:42:59,744 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/39148337aa4f4f07ae5e411f22b85333 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/39148337aa4f4f07ae5e411f22b85333 2024-11-18T20:42:59,746 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/a10e344e29684d1487633ef2f6e0c6eb to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/a10e344e29684d1487633ef2f6e0c6eb 2024-11-18T20:42:59,748 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/d2131f09a9604d15b25d1f3b97c93b77 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/A/d2131f09a9604d15b25d1f3b97c93b77 2024-11-18T20:42:59,752 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/bb423c5b8971420681277365facb63c7 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/bb423c5b8971420681277365facb63c7 2024-11-18T20:42:59,754 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/d521785dc6fb4226b7cb48430a6e17f3 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/d521785dc6fb4226b7cb48430a6e17f3 2024-11-18T20:42:59,756 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/fb78533f70f44611b860f49baf7a37bd to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/B/fb78533f70f44611b860f49baf7a37bd 2024-11-18T20:42:59,759 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/36415b960f2748f295cf30e2a1d138f1 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/36415b960f2748f295cf30e2a1d138f1 2024-11-18T20:42:59,760 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/4fcaaec6d9ef4073b0b5543961486b9e to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/4fcaaec6d9ef4073b0b5543961486b9e 2024-11-18T20:42:59,762 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/a3099ccbeb454d7d99e80b871db77d41 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/C/a3099ccbeb454d7d99e80b871db77d41 2024-11-18T20:42:59,765 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/recovered.edits/268.seqid to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223/recovered.edits/268.seqid 2024-11-18T20:42:59,766 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/75114f181d047350ddda984d66f6e223 2024-11-18T20:42:59,766 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T20:42:59,766 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T20:42:59,767 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-18T20:42:59,770 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118208969f0e9ae4bcda1a8ce67193396bf_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118208969f0e9ae4bcda1a8ce67193396bf_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:59,772 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183c039d83f7ca420c81ac1508ea0794ab_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183c039d83f7ca420c81ac1508ea0794ab_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:59,774 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111843292ca8beb646f7ad21889e02583c14_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111843292ca8beb646f7ad21889e02583c14_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:59,775 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118436a862c64954921af896f8e028e09d2_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118436a862c64954921af896f8e028e09d2_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:59,777 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184b08380ea3ba43a9859075d6b8b19a85_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184b08380ea3ba43a9859075d6b8b19a85_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:59,778 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185f2a68cd719843bd8bbd7460b1128aa7_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185f2a68cd719843bd8bbd7460b1128aa7_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:59,779 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187a6487fa1d274e93a87982179e0997ba_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187a6487fa1d274e93a87982179e0997ba_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:59,781 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118933c18f1237a429c902e69fa0f66480e_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118933c18f1237a429c902e69fa0f66480e_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:59,783 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118939afbf848bd46869eadb9623735f412_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118939afbf848bd46869eadb9623735f412_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:59,784 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189cb3fc4823444f588be33c304231e581_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189cb3fc4823444f588be33c304231e581_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:59,785 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b65101d2f5154c8886557fd1b1b9270a_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b65101d2f5154c8886557fd1b1b9270a_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:59,786 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c36610bac6fa4cdb95f2d84cf5aa5ca1_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c36610bac6fa4cdb95f2d84cf5aa5ca1_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:59,787 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c5c29028fa0c44fca944b673e59e4820_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c5c29028fa0c44fca944b673e59e4820_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:59,788 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d6528e73feb64294a688186e886c4bea_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d6528e73feb64294a688186e886c4bea_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:59,790 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ee96bf45da204aaeabd4829382fbb569_75114f181d047350ddda984d66f6e223 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ee96bf45da204aaeabd4829382fbb569_75114f181d047350ddda984d66f6e223 2024-11-18T20:42:59,790 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T20:42:59,792 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:42:59,794 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T20:42:59,796 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T20:42:59,797 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:42:59,798 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T20:42:59,798 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731962579798"}]},"ts":"9223372036854775807"} 2024-11-18T20:42:59,800 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T20:42:59,800 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 75114f181d047350ddda984d66f6e223, NAME => 'TestAcidGuarantees,,1731962546273.75114f181d047350ddda984d66f6e223.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T20:42:59,800 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T20:42:59,800 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731962579800"}]},"ts":"9223372036854775807"} 2024-11-18T20:42:59,802 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T20:42:59,803 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:42:59,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 74 msec 2024-11-18T20:42:59,825 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T20:42:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T20:42:59,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T20:42:59,837 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:42:59,846 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 246) - Thread LEAK? -, OpenFileDescriptor=443 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=372 (was 389), ProcessCount=11 (was 11), AvailableMemoryMB=1041 (was 1612) 2024-11-18T20:42:59,854 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=372, ProcessCount=11, AvailableMemoryMB=1041 2024-11-18T20:42:59,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:42:59,855 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:42:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T20:42:59,858 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T20:42:59,858 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:42:59,858 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-18T20:42:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-18T20:42:59,859 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T20:42:59,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742258_1434 (size=1061) 2024-11-18T20:42:59,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-18T20:43:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-18T20:43:00,267 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093 2024-11-18T20:43:00,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742259_1435 (size=53) 2024-11-18T20:43:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-18T20:43:00,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:43:00,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 22cccb73b6312f1ad45031615ae4847f, disabling compactions & flushes 2024-11-18T20:43:00,676 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:00,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:00,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. after waiting 0 ms 2024-11-18T20:43:00,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:00,676 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:00,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 22cccb73b6312f1ad45031615ae4847f: Waiting for close lock at 1731962580676Disabling compacts and flushes for region at 1731962580676Disabling writes for close at 1731962580676Writing region close event to WAL at 1731962580676Closed at 1731962580676 2024-11-18T20:43:00,678 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T20:43:00,678 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731962580678"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731962580678"}]},"ts":"1731962580678"} 2024-11-18T20:43:00,680 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T20:43:00,681 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T20:43:00,681 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962580681"}]},"ts":"1731962580681"} 2024-11-18T20:43:00,683 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T20:43:00,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22cccb73b6312f1ad45031615ae4847f, ASSIGN}] 2024-11-18T20:43:00,686 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22cccb73b6312f1ad45031615ae4847f, ASSIGN 2024-11-18T20:43:00,687 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22cccb73b6312f1ad45031615ae4847f, ASSIGN; state=OFFLINE, location=c0a89b2656d4,45393,1731962440107; forceNewPlan=false, retain=false 2024-11-18T20:43:00,838 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=22cccb73b6312f1ad45031615ae4847f, regionState=OPENING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:00,841 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22cccb73b6312f1ad45031615ae4847f, ASSIGN because future has completed 2024-11-18T20:43:00,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:43:00,997 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-18T20:43:00,997 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:43:00,998 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:00,998 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:43:00,998 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:00,998 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:00,999 INFO [StoreOpener-22cccb73b6312f1ad45031615ae4847f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:01,001 INFO [StoreOpener-22cccb73b6312f1ad45031615ae4847f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:43:01,001 INFO [StoreOpener-22cccb73b6312f1ad45031615ae4847f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22cccb73b6312f1ad45031615ae4847f columnFamilyName A 2024-11-18T20:43:01,001 DEBUG [StoreOpener-22cccb73b6312f1ad45031615ae4847f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:01,002 INFO [StoreOpener-22cccb73b6312f1ad45031615ae4847f-1 {}] regionserver.HStore(327): Store=22cccb73b6312f1ad45031615ae4847f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:43:01,002 INFO [StoreOpener-22cccb73b6312f1ad45031615ae4847f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:01,003 INFO [StoreOpener-22cccb73b6312f1ad45031615ae4847f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:43:01,004 INFO [StoreOpener-22cccb73b6312f1ad45031615ae4847f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22cccb73b6312f1ad45031615ae4847f columnFamilyName B 2024-11-18T20:43:01,004 DEBUG [StoreOpener-22cccb73b6312f1ad45031615ae4847f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:01,004 INFO [StoreOpener-22cccb73b6312f1ad45031615ae4847f-1 {}] regionserver.HStore(327): Store=22cccb73b6312f1ad45031615ae4847f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:43:01,004 INFO [StoreOpener-22cccb73b6312f1ad45031615ae4847f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:01,005 INFO [StoreOpener-22cccb73b6312f1ad45031615ae4847f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:43:01,005 INFO [StoreOpener-22cccb73b6312f1ad45031615ae4847f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22cccb73b6312f1ad45031615ae4847f columnFamilyName C 2024-11-18T20:43:01,005 DEBUG [StoreOpener-22cccb73b6312f1ad45031615ae4847f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:01,006 INFO [StoreOpener-22cccb73b6312f1ad45031615ae4847f-1 {}] regionserver.HStore(327): Store=22cccb73b6312f1ad45031615ae4847f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:43:01,006 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:01,006 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:01,007 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:01,007 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:01,008 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:01,008 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:01,008 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:43:01,010 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:01,011 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:43:01,012 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened 22cccb73b6312f1ad45031615ae4847f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75234247, jitterRate=0.12107764184474945}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:43:01,012 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:01,012 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for 22cccb73b6312f1ad45031615ae4847f: Running coprocessor pre-open hook at 1731962580998Writing region info on filesystem at 1731962580998Initializing all the Stores at 1731962580999 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962580999Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962580999Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962580999Cleaning up temporary data from old regions at 1731962581008 (+9 ms)Running coprocessor post-open hooks at 1731962581012 (+4 ms)Region opened successfully at 1731962581012 2024-11-18T20:43:01,013 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., pid=118, masterSystemTime=1731962580993 2024-11-18T20:43:01,015 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:01,015 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:01,016 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=22cccb73b6312f1ad45031615ae4847f, regionState=OPEN, openSeqNum=2, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:01,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 because future has completed 2024-11-18T20:43:01,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-18T20:43:01,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure 22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 in 177 msec 2024-11-18T20:43:01,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-18T20:43:01,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22cccb73b6312f1ad45031615ae4847f, ASSIGN in 336 msec 2024-11-18T20:43:01,023 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T20:43:01,023 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962581023"}]},"ts":"1731962581023"} 2024-11-18T20:43:01,024 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T20:43:01,025 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T20:43:01,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1700 sec 2024-11-18T20:43:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-18T20:43:02,008 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:02,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c7b275, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,009 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:02,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:02,010 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:02,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:02,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:02,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29d4939a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:02,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:02,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:02,012 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:02,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@209534fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,013 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:02,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:02,016 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:43:02,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is c0a89b2656d4,46825,1731962439484 2024-11-18T20:43:02,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@de824ec 2024-11-18T20:43:02,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:43:02,018 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:43:02,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fcbf455, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,019 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:02,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:02,020 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:02,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:02,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:02,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22efaa0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:02,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:02,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:02,022 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:02,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f18b04e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc683cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:02,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:02,024 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:02,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:02,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:02,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26f86353, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:02,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:02,025 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:02,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@97c4ea2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43681357, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:02,027 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:02,027 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e1dbeda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:02,029 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:02,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@717c9dbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39982088, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,030 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:02,030 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:02,030 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dfaaf09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:02,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:02,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:02,031 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:02,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a4a716d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e4c85de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,032 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:02,032 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:02,033 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d5fd659, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:02,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:02,034 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:02,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da65617, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,035 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44ce80f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,035 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:02,035 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:02,036 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:02,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:02,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:02,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@132bf41c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:02,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:02,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:02,037 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:02,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68fd8e07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@125833c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,038 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:02,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:02,039 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:02,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:02,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:02,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51377d1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:02,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:02,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:02,040 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:02,041 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@651e5ac4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea16345, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,042 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:02,042 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:02,042 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@179972dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:02,044 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:02,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@116d594e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2334b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,048 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:02,048 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:02,049 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aaba15f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:02,050 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71f9c86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d1af8a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,051 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:02,052 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:02,052 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:02,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5661f2e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:02,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:02,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:02,053 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:02,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cc2507e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:02,055 DEBUG [Thread-1980 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:02,055 DEBUG [Thread-1963 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:02,055 DEBUG [Thread-1971 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:02,055 DEBUG [Thread-1978 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:02,055 DEBUG [Thread-1965 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:02,055 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:43:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:02,056 DEBUG [Thread-1969 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-18T20:43:02,056 DEBUG [Thread-1974 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:02,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:02,057 DEBUG [Thread-1976 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:02,057 DEBUG [Thread-1982 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:02,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:02,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:02,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:02,058 DEBUG [Thread-1967 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:02,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:02,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:02,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:02,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:02,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:02,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T20:43:02,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:02,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:02,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:02,059 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:43:02,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:02,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:02,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:02,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:02,060 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:43:02,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:43:02,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:43:02,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:43:02,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:43:02,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:43:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:43:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:43:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:02,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:02,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:43:02,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:43:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:43:02,069 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2] 2024-11-18T20:43:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:02,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T20:43:02,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:02,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:02,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:02,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:02,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:02,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:02,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/66712570f2b34b2fbeb0dd820329669d is 50, key is test_row_0/A:col10/1731962582064/Put/seqid=0 2024-11-18T20:43:02,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742260_1436 (size=12001) 2024-11-18T20:43:02,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/66712570f2b34b2fbeb0dd820329669d 2024-11-18T20:43:02,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962582100, max timestamp=1731962582114Num uniques 859; 2024-11-18T20:43:02,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:43:02,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962582100, max timestamp=1731962582114Num uniques 859; 2024-11-18T20:43:02,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:43:02,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962582100, max timestamp=1731962582114Num uniques 859; 2024-11-18T20:43:02,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:43:02,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/b19375e7e6d44c95a084b9d67cb19fc5 is 50, key is test_row_0/B:col10/1731962582064/Put/seqid=0 2024-11-18T20:43:02,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742261_1437 (size=12001) 2024-11-18T20:43:02,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962642148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962642149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962642150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962642154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962642155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T20:43:02,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T20:43:02,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:02,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:02,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:02,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:02,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:02,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962642267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962642267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962642268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962642268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962642268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T20:43:02,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:02,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T20:43:02,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962642478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962642478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962642479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962642479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962642479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T20:43:02,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:02,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:02,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:02,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:02,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/b19375e7e6d44c95a084b9d67cb19fc5 2024-11-18T20:43:02,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/555a438f55234b02ab10d62c91bb984e is 50, key is test_row_0/C:col10/1731962582064/Put/seqid=0 2024-11-18T20:43:02,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742262_1438 (size=12001) 2024-11-18T20:43:02,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T20:43:02,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:02,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:02,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:02,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:02,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:02,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T20:43:02,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962642797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962642797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962642798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962642798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:02,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962642798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:02,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:02,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:02,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T20:43:02,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:02,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:02,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:02,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:02,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:02,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:02,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/555a438f55234b02ab10d62c91bb984e 2024-11-18T20:43:02,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/66712570f2b34b2fbeb0dd820329669d as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/66712570f2b34b2fbeb0dd820329669d 2024-11-18T20:43:02,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T20:43:02,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:02,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:02,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:02,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:02,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/66712570f2b34b2fbeb0dd820329669d, entries=150, sequenceid=11, filesize=11.7 K 2024-11-18T20:43:02,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:02,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/b19375e7e6d44c95a084b9d67cb19fc5 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/b19375e7e6d44c95a084b9d67cb19fc5 2024-11-18T20:43:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:02,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/b19375e7e6d44c95a084b9d67cb19fc5, entries=150, sequenceid=11, filesize=11.7 K 2024-11-18T20:43:02,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/555a438f55234b02ab10d62c91bb984e as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/555a438f55234b02ab10d62c91bb984e 2024-11-18T20:43:02,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/555a438f55234b02ab10d62c91bb984e, entries=150, sequenceid=11, filesize=11.7 K 2024-11-18T20:43:02,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 22cccb73b6312f1ad45031615ae4847f in 915ms, sequenceid=11, compaction requested=false 2024-11-18T20:43:02,989 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T20:43:02,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:03,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T20:43:03,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:03,132 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-18T20:43:03,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:03,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:03,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:03,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:03,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:03,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:03,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/b1f4e3b47dc84346bec2c4e8bcd6b64c is 50, key is test_row_0/A:col10/1731962582148/Put/seqid=0 2024-11-18T20:43:03,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742263_1439 (size=12001) 2024-11-18T20:43:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T20:43:03,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:03,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:03,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962643318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962643318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:03,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962643326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:03,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962643327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:03,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962643328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:03,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962643437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:03,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962643437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:03,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962643447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:03,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962643447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:03,543 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/b1f4e3b47dc84346bec2c4e8bcd6b64c 2024-11-18T20:43:03,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/7fdc81ca77314da89636cd7d69cfcaa3 is 50, key is test_row_0/B:col10/1731962582148/Put/seqid=0 2024-11-18T20:43:03,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742264_1440 (size=12001) 2024-11-18T20:43:03,559 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/7fdc81ca77314da89636cd7d69cfcaa3 2024-11-18T20:43:03,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/c0eb2075dfa245aebe186f34d81c212b is 50, key is test_row_0/C:col10/1731962582148/Put/seqid=0 2024-11-18T20:43:03,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742265_1441 (size=12001) 2024-11-18T20:43:03,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962643657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:03,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962643657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:03,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:03,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:03,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962643667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962643667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:03,975 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/c0eb2075dfa245aebe186f34d81c212b 2024-11-18T20:43:03,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/b1f4e3b47dc84346bec2c4e8bcd6b64c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/b1f4e3b47dc84346bec2c4e8bcd6b64c 2024-11-18T20:43:03,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:03,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962643977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:03,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:03,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962643977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:03,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962643977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:03,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962643978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:03,983 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/b1f4e3b47dc84346bec2c4e8bcd6b64c, entries=150, sequenceid=48, filesize=11.7 K 2024-11-18T20:43:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:03,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/7fdc81ca77314da89636cd7d69cfcaa3 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/7fdc81ca77314da89636cd7d69cfcaa3 2024-11-18T20:43:03,987 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/7fdc81ca77314da89636cd7d69cfcaa3, entries=150, sequenceid=48, filesize=11.7 K 2024-11-18T20:43:03,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/c0eb2075dfa245aebe186f34d81c212b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/c0eb2075dfa245aebe186f34d81c212b 2024-11-18T20:43:03,992 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/c0eb2075dfa245aebe186f34d81c212b, entries=150, sequenceid=48, filesize=11.7 K 2024-11-18T20:43:03,992 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 22cccb73b6312f1ad45031615ae4847f in 860ms, sequenceid=48, compaction requested=false 2024-11-18T20:43:03,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:03,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:03,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-18T20:43:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-18T20:43:03,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-18T20:43:03,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9340 sec 2024-11-18T20:43:03,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.9410 sec 2024-11-18T20:43:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T20:43:04,207 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:04,207 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:43:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-18T20:43:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T20:43:04,209 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:43:04,209 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:43:04,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:43:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T20:43:04,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:04,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T20:43:04,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:04,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:04,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:04,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:04,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:04,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:04,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/6f969d5582234a9893be2626c6c63194 is 50, key is test_row_0/A:col10/1731962583315/Put/seqid=0 2024-11-18T20:43:04,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742266_1442 (size=14341) 2024-11-18T20:43:04,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/6f969d5582234a9893be2626c6c63194 2024-11-18T20:43:04,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/60fd0e036c1f484f842ad8eb580bd425 is 50, key is test_row_0/B:col10/1731962583315/Put/seqid=0 2024-11-18T20:43:04,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742267_1443 (size=12001) 2024-11-18T20:43:04,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:43:04,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:04,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:04,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:04,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:04,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:04,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:04,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962584464, max timestamp=1731962584499Num uniques 859; 2024-11-18T20:43:04,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:43:04,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962584464, max timestamp=1731962584499Num uniques 859; 2024-11-18T20:43:04,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:43:04,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962584464, max timestamp=1731962584499Num uniques 859; 2024-11-18T20:43:04,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:43:04,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:43:04,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:04,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:04,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:04,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:04,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T20:43:04,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962644544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:04,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962644546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:04,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962644547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:04,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962644547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:04,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:04,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962644548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:04,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:04,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:04,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962644657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:04,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:43:04,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:04,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:04,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:04,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:04,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962644668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:04,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962644668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:04,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962644668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962644677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:04,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/60fd0e036c1f484f842ad8eb580bd425 2024-11-18T20:43:04,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/399e1fffbe944c06889b26f4cb5fb5a6 is 50, key is test_row_0/C:col10/1731962583315/Put/seqid=0 2024-11-18T20:43:04,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742268_1444 (size=12001) 2024-11-18T20:43:04,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:43:04,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:04,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:04,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:04,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:04,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:04,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T20:43:04,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:04,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962644877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:04,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:04,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962644887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:04,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962644887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:04,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:04,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962644887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:04,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962644888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:04,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:04,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:04,971 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T20:43:04,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:43:04,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:04,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:04,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:04,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:04,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:04,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:43:05,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:05,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:05,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:05,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/399e1fffbe944c06889b26f4cb5fb5a6 2024-11-18T20:43:05,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/6f969d5582234a9893be2626c6c63194 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/6f969d5582234a9893be2626c6c63194 2024-11-18T20:43:05,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/6f969d5582234a9893be2626c6c63194, entries=200, sequenceid=59, filesize=14.0 K 2024-11-18T20:43:05,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/60fd0e036c1f484f842ad8eb580bd425 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/60fd0e036c1f484f842ad8eb580bd425 2024-11-18T20:43:05,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/60fd0e036c1f484f842ad8eb580bd425, entries=150, sequenceid=59, filesize=11.7 K 2024-11-18T20:43:05,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/399e1fffbe944c06889b26f4cb5fb5a6 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/399e1fffbe944c06889b26f4cb5fb5a6 2024-11-18T20:43:05,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/399e1fffbe944c06889b26f4cb5fb5a6, entries=150, sequenceid=59, filesize=11.7 K 2024-11-18T20:43:05,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 22cccb73b6312f1ad45031615ae4847f in 846ms, sequenceid=59, compaction requested=true 2024-11-18T20:43:05,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:05,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:43:05,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:05,186 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:05,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:43:05,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:05,186 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:05,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:43:05,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:05,187 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:05,187 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:05,187 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/A is initiating minor compaction (all files) 2024-11-18T20:43:05,187 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/B is initiating minor compaction (all files) 2024-11-18T20:43:05,187 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/A in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:05,187 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/B in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:05,187 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/b19375e7e6d44c95a084b9d67cb19fc5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/7fdc81ca77314da89636cd7d69cfcaa3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/60fd0e036c1f484f842ad8eb580bd425] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=35.2 K 2024-11-18T20:43:05,187 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/66712570f2b34b2fbeb0dd820329669d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/b1f4e3b47dc84346bec2c4e8bcd6b64c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/6f969d5582234a9893be2626c6c63194] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=37.4 K 2024-11-18T20:43:05,187 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting b19375e7e6d44c95a084b9d67cb19fc5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731962582062 2024-11-18T20:43:05,188 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fdc81ca77314da89636cd7d69cfcaa3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731962582148 2024-11-18T20:43:05,188 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66712570f2b34b2fbeb0dd820329669d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731962582062 2024-11-18T20:43:05,188 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 60fd0e036c1f484f842ad8eb580bd425, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731962583315 2024-11-18T20:43:05,188 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1f4e3b47dc84346bec2c4e8bcd6b64c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731962582148 2024-11-18T20:43:05,188 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f969d5582234a9893be2626c6c63194, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731962583315 2024-11-18T20:43:05,196 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#B#compaction#374 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:05,197 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/f5c0ceb7ef564c7cafb0eecfeae0397c is 50, key is test_row_0/B:col10/1731962583315/Put/seqid=0 2024-11-18T20:43:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:05,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-18T20:43:05,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:05,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:05,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:05,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:05,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:05,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:05,207 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#A#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:05,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/21d34c613cb6453a9b75f7a193df17c2 is 50, key is test_row_0/A:col10/1731962585199/Put/seqid=0 2024-11-18T20:43:05,208 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/a949afb50ef244a3bfca95ee227fd556 is 50, key is test_row_0/A:col10/1731962583315/Put/seqid=0 2024-11-18T20:43:05,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742269_1445 (size=12104) 2024-11-18T20:43:05,216 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/f5c0ceb7ef564c7cafb0eecfeae0397c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/f5c0ceb7ef564c7cafb0eecfeae0397c 2024-11-18T20:43:05,221 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/B of 22cccb73b6312f1ad45031615ae4847f into f5c0ceb7ef564c7cafb0eecfeae0397c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:05,221 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:05,221 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/B, priority=13, startTime=1731962585186; duration=0sec 2024-11-18T20:43:05,221 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:05,221 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:B 2024-11-18T20:43:05,221 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:05,222 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:05,222 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/C is initiating minor compaction (all files) 2024-11-18T20:43:05,222 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/C in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:05,222 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/555a438f55234b02ab10d62c91bb984e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/c0eb2075dfa245aebe186f34d81c212b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/399e1fffbe944c06889b26f4cb5fb5a6] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=35.2 K 2024-11-18T20:43:05,222 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 555a438f55234b02ab10d62c91bb984e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731962582062 2024-11-18T20:43:05,222 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting c0eb2075dfa245aebe186f34d81c212b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731962582148 2024-11-18T20:43:05,222 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 399e1fffbe944c06889b26f4cb5fb5a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731962583315 2024-11-18T20:43:05,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962645218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742270_1446 (size=14341) 2024-11-18T20:43:05,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742271_1447 (size=12104) 2024-11-18T20:43:05,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/21d34c613cb6453a9b75f7a193df17c2 2024-11-18T20:43:05,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962645218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962645220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962645226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,237 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#C#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:05,237 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/04b92846b9ac4dc394104ec32d8981c2 is 50, key is test_row_0/C:col10/1731962583315/Put/seqid=0 2024-11-18T20:43:05,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742272_1448 (size=12104) 2024-11-18T20:43:05,241 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/a949afb50ef244a3bfca95ee227fd556 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/a949afb50ef244a3bfca95ee227fd556 2024-11-18T20:43:05,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/53ee90ba07394d17812f811269297268 is 50, key is test_row_0/B:col10/1731962585199/Put/seqid=0 2024-11-18T20:43:05,246 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/A of 22cccb73b6312f1ad45031615ae4847f into a949afb50ef244a3bfca95ee227fd556(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:05,246 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:05,246 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/A, priority=13, startTime=1731962585186; duration=0sec 2024-11-18T20:43:05,246 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:05,246 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:A 2024-11-18T20:43:05,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742273_1449 (size=12001) 2024-11-18T20:43:05,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962645231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:43:05,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:05,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:05,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:05,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T20:43:05,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962645337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962645347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962645347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962645347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962645357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:43:05,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:05,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:05,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:05,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962645557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962645567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962645567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962645568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962645577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:43:05,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:05,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:05,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:05,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,644 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/04b92846b9ac4dc394104ec32d8981c2 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/04b92846b9ac4dc394104ec32d8981c2 2024-11-18T20:43:05,648 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/C of 22cccb73b6312f1ad45031615ae4847f into 04b92846b9ac4dc394104ec32d8981c2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:05,648 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:05,648 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/C, priority=13, startTime=1731962585186; duration=0sec 2024-11-18T20:43:05,648 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:05,648 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:C 2024-11-18T20:43:05,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/53ee90ba07394d17812f811269297268 2024-11-18T20:43:05,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/556f05a306ce4aaca60e1a14d408efe5 is 50, key is test_row_0/C:col10/1731962585199/Put/seqid=0 2024-11-18T20:43:05,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742274_1450 (size=12001) 2024-11-18T20:43:05,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:43:05,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:05,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:05,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:05,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962645877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:43:05,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:05,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:05,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:05,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:05,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962645887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962645887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962645887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:05,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962645897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:06,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:43:06,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:06,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:06,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:06,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:06,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:06,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/556f05a306ce4aaca60e1a14d408efe5 2024-11-18T20:43:06,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/21d34c613cb6453a9b75f7a193df17c2 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/21d34c613cb6453a9b75f7a193df17c2 2024-11-18T20:43:06,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/21d34c613cb6453a9b75f7a193df17c2, entries=200, sequenceid=95, filesize=14.0 K 2024-11-18T20:43:06,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/53ee90ba07394d17812f811269297268 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/53ee90ba07394d17812f811269297268 2024-11-18T20:43:06,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/53ee90ba07394d17812f811269297268, entries=150, sequenceid=95, filesize=11.7 K 2024-11-18T20:43:06,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/556f05a306ce4aaca60e1a14d408efe5 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/556f05a306ce4aaca60e1a14d408efe5 2024-11-18T20:43:06,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/556f05a306ce4aaca60e1a14d408efe5, entries=150, sequenceid=95, filesize=11.7 K 2024-11-18T20:43:06,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 22cccb73b6312f1ad45031615ae4847f in 875ms, sequenceid=95, compaction requested=false 2024-11-18T20:43:06,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:06,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:43:06,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:06,200 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T20:43:06,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:06,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:06,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:06,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:06,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:06,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:06,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/9f108fafdff742af8c67f387fb4b5e48 is 50, key is test_row_0/A:col10/1731962585215/Put/seqid=0 2024-11-18T20:43:06,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742275_1451 (size=12001) 2024-11-18T20:43:06,216 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/9f108fafdff742af8c67f387fb4b5e48 2024-11-18T20:43:06,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/810258417f864e43b4a162595ea8e8b3 is 50, key is test_row_0/B:col10/1731962585215/Put/seqid=0 2024-11-18T20:43:06,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742276_1452 (size=12001) 2024-11-18T20:43:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T20:43:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:06,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:06,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962586426, max timestamp=1731962586451Num uniques 859; 2024-11-18T20:43:06,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962586426, max timestamp=1731962586451Num uniques 859; 2024-11-18T20:43:06,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:43:06,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:43:06,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962586426, max timestamp=1731962586451Num uniques 859; 2024-11-18T20:43:06,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:43:06,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962646492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:06,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962646492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:06,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:06,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:06,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962646494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962646493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:06,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962646495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:06,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962646597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:06,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962646617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:06,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962646617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962646617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962646618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,626 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/810258417f864e43b4a162595ea8e8b3 2024-11-18T20:43:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:06,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/21c34857a9fa41d0a0b2a59e5cc7c0e0 is 50, key is test_row_0/C:col10/1731962585215/Put/seqid=0 2024-11-18T20:43:06,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742277_1453 (size=12001) 2024-11-18T20:43:06,636 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/21c34857a9fa41d0a0b2a59e5cc7c0e0 2024-11-18T20:43:06,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/9f108fafdff742af8c67f387fb4b5e48 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/9f108fafdff742af8c67f387fb4b5e48 2024-11-18T20:43:06,643 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/9f108fafdff742af8c67f387fb4b5e48, entries=150, sequenceid=108, filesize=11.7 K 2024-11-18T20:43:06,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/810258417f864e43b4a162595ea8e8b3 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/810258417f864e43b4a162595ea8e8b3 2024-11-18T20:43:06,646 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/810258417f864e43b4a162595ea8e8b3, entries=150, sequenceid=108, filesize=11.7 K 2024-11-18T20:43:06,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/21c34857a9fa41d0a0b2a59e5cc7c0e0 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/21c34857a9fa41d0a0b2a59e5cc7c0e0 2024-11-18T20:43:06,650 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/21c34857a9fa41d0a0b2a59e5cc7c0e0, entries=150, sequenceid=108, filesize=11.7 K 2024-11-18T20:43:06,651 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 22cccb73b6312f1ad45031615ae4847f in 451ms, sequenceid=108, compaction requested=true 2024-11-18T20:43:06,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:06,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:06,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-18T20:43:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-18T20:43:06,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-18T20:43:06,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4430 sec 2024-11-18T20:43:06,656 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.4480 sec 2024-11-18T20:43:06,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:06,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-18T20:43:06,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:06,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:06,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:06,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:06,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:06,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:06,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/28806fbec4884dfeacb0d3a82eaa4bcb is 50, key is test_row_0/A:col10/1731962586491/Put/seqid=0 2024-11-18T20:43:06,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742278_1454 (size=14541) 2024-11-18T20:43:06,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962646837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:06,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962646837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:06,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962646843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:06,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962646844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:06,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962646847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:06,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962646957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:06,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:06,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962646967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962646967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:07,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962647157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962647157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:07,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962647177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:07,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962647177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962647177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:07,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/28806fbec4884dfeacb0d3a82eaa4bcb 2024-11-18T20:43:07,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/7628568f76e64acc966a856637fcd899 is 50, key is test_row_0/B:col10/1731962586491/Put/seqid=0 2024-11-18T20:43:07,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742279_1455 (size=12151) 2024-11-18T20:43:07,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/7628568f76e64acc966a856637fcd899 2024-11-18T20:43:07,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/9b830bb5e32846789a0f2addfcda1e01 is 50, key is test_row_0/C:col10/1731962586491/Put/seqid=0 2024-11-18T20:43:07,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742280_1456 (size=12151) 2024-11-18T20:43:07,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/9b830bb5e32846789a0f2addfcda1e01 2024-11-18T20:43:07,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/28806fbec4884dfeacb0d3a82eaa4bcb as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/28806fbec4884dfeacb0d3a82eaa4bcb 2024-11-18T20:43:07,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/28806fbec4884dfeacb0d3a82eaa4bcb, entries=200, sequenceid=145, filesize=14.2 K 2024-11-18T20:43:07,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/7628568f76e64acc966a856637fcd899 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/7628568f76e64acc966a856637fcd899 2024-11-18T20:43:07,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/7628568f76e64acc966a856637fcd899, entries=150, sequenceid=145, filesize=11.9 K 2024-11-18T20:43:07,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/9b830bb5e32846789a0f2addfcda1e01 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/9b830bb5e32846789a0f2addfcda1e01 2024-11-18T20:43:07,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/9b830bb5e32846789a0f2addfcda1e01, entries=150, sequenceid=145, filesize=11.9 K 2024-11-18T20:43:07,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 22cccb73b6312f1ad45031615ae4847f in 473ms, sequenceid=145, compaction requested=true 2024-11-18T20:43:07,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:07,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:43:07,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:07,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:43:07,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:07,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:43:07,294 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:43:07,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:07,294 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:43:07,295 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52987 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:43:07,295 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:43:07,295 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/B is initiating minor compaction (all files) 2024-11-18T20:43:07,295 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/A is initiating minor compaction (all files) 2024-11-18T20:43:07,295 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/A in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:07,295 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/B in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:07,295 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/f5c0ceb7ef564c7cafb0eecfeae0397c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/53ee90ba07394d17812f811269297268, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/810258417f864e43b4a162595ea8e8b3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/7628568f76e64acc966a856637fcd899] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=47.1 K 2024-11-18T20:43:07,295 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/a949afb50ef244a3bfca95ee227fd556, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/21d34c613cb6453a9b75f7a193df17c2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/9f108fafdff742af8c67f387fb4b5e48, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/28806fbec4884dfeacb0d3a82eaa4bcb] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=51.7 K 2024-11-18T20:43:07,295 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting f5c0ceb7ef564c7cafb0eecfeae0397c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731962583315 2024-11-18T20:43:07,295 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting a949afb50ef244a3bfca95ee227fd556, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731962583315 2024-11-18T20:43:07,296 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 53ee90ba07394d17812f811269297268, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731962584545 2024-11-18T20:43:07,296 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21d34c613cb6453a9b75f7a193df17c2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731962584545 2024-11-18T20:43:07,296 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 810258417f864e43b4a162595ea8e8b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731962585215 2024-11-18T20:43:07,296 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f108fafdff742af8c67f387fb4b5e48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731962585215 2024-11-18T20:43:07,296 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 7628568f76e64acc966a856637fcd899, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731962586491 2024-11-18T20:43:07,297 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28806fbec4884dfeacb0d3a82eaa4bcb, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731962586491 2024-11-18T20:43:07,305 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#B#compaction#386 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:07,305 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#A#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:07,305 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/eeb2ec43f6a14bc58381b3b67db0cbb9 is 50, key is test_row_0/B:col10/1731962586491/Put/seqid=0 2024-11-18T20:43:07,305 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/1bdf7aff363a4425900239cfe31a3770 is 50, key is test_row_0/A:col10/1731962586491/Put/seqid=0 2024-11-18T20:43:07,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742281_1457 (size=12391) 2024-11-18T20:43:07,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742282_1458 (size=12391) 2024-11-18T20:43:07,319 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/1bdf7aff363a4425900239cfe31a3770 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/1bdf7aff363a4425900239cfe31a3770 2024-11-18T20:43:07,323 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/A of 22cccb73b6312f1ad45031615ae4847f into 1bdf7aff363a4425900239cfe31a3770(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:07,323 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:07,323 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/A, priority=12, startTime=1731962587293; duration=0sec 2024-11-18T20:43:07,323 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:07,324 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:A 2024-11-18T20:43:07,324 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:43:07,327 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:43:07,327 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/C is initiating minor compaction (all files) 2024-11-18T20:43:07,327 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/C in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:07,327 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/04b92846b9ac4dc394104ec32d8981c2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/556f05a306ce4aaca60e1a14d408efe5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/21c34857a9fa41d0a0b2a59e5cc7c0e0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/9b830bb5e32846789a0f2addfcda1e01] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=47.1 K 2024-11-18T20:43:07,327 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04b92846b9ac4dc394104ec32d8981c2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731962583315 2024-11-18T20:43:07,328 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 556f05a306ce4aaca60e1a14d408efe5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731962584545 2024-11-18T20:43:07,328 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21c34857a9fa41d0a0b2a59e5cc7c0e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731962585215 2024-11-18T20:43:07,328 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b830bb5e32846789a0f2addfcda1e01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731962586491 2024-11-18T20:43:07,334 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#C#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:07,334 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/2596a6a0a0994a1da21f10ac6d12f1b2 is 50, key is test_row_0/C:col10/1731962586491/Put/seqid=0 2024-11-18T20:43:07,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742283_1459 (size=12391) 2024-11-18T20:43:07,346 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/2596a6a0a0994a1da21f10ac6d12f1b2 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/2596a6a0a0994a1da21f10ac6d12f1b2 2024-11-18T20:43:07,351 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/C of 22cccb73b6312f1ad45031615ae4847f into 2596a6a0a0994a1da21f10ac6d12f1b2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:07,351 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:07,351 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/C, priority=12, startTime=1731962587294; duration=0sec 2024-11-18T20:43:07,351 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:07,351 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:C 2024-11-18T20:43:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:07,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T20:43:07,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:07,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:07,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:07,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:07,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:07,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:07,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/4404fdefe1bf47a4ac66f4731ed862b2 is 50, key is test_row_0/A:col10/1731962586845/Put/seqid=0 2024-11-18T20:43:07,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742284_1460 (size=12151) 2024-11-18T20:43:07,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962587517, max timestamp=1731962587552Num uniques 859; 2024-11-18T20:43:07,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:43:07,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962587517, max timestamp=1731962587552Num uniques 859; 2024-11-18T20:43:07,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:43:07,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962587517, max timestamp=1731962587552Num uniques 859; 2024-11-18T20:43:07,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:43:07,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962647629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:07,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962647639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:07,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962647640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:07,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962647677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:07,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962647677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:07,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:07,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:07,717 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/eeb2ec43f6a14bc58381b3b67db0cbb9 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/eeb2ec43f6a14bc58381b3b67db0cbb9 2024-11-18T20:43:07,722 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/B of 22cccb73b6312f1ad45031615ae4847f into eeb2ec43f6a14bc58381b3b67db0cbb9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:07,722 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:07,722 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/B, priority=12, startTime=1731962587294; duration=0sec 2024-11-18T20:43:07,722 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:07,722 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:B 2024-11-18T20:43:07,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:07,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962647747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:07,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:07,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962647767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:07,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:07,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962647767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:07,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:07,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:07,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/4404fdefe1bf47a4ac66f4731ed862b2 2024-11-18T20:43:07,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/fd1a71fcde674caaa55a35b2644d78df is 50, key is test_row_0/B:col10/1731962586845/Put/seqid=0 2024-11-18T20:43:07,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742285_1461 (size=12151) 2024-11-18T20:43:07,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:07,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962647957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:07,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:07,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:07,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962647977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:07,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:07,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962647978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:08,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:08,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962648267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:08,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962648297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962648297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:08,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/fd1a71fcde674caaa55a35b2644d78df 2024-11-18T20:43:08,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/9a352d45fa034f62bce432b77b111549 is 50, key is test_row_0/C:col10/1731962586845/Put/seqid=0 2024-11-18T20:43:08,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742286_1462 (size=12151) 2024-11-18T20:43:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T20:43:08,377 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:08,378 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:43:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-18T20:43:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:43:08,380 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:43:08,380 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:43:08,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:43:08,409 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:43:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:43:08,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:43:08,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:08,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:08,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:08,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:43:08,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:08,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:08,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:08,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:08,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:08,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:08,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:43:08,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962648697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:08,708 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 22cccb73b6312f1ad45031615ae4847f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:08,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:08,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962648708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:08,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/9a352d45fa034f62bce432b77b111549 2024-11-18T20:43:08,719 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in 22cccb73b6312f1ad45031615ae4847f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:08,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/4404fdefe1bf47a4ac66f4731ed862b2 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/4404fdefe1bf47a4ac66f4731ed862b2 2024-11-18T20:43:08,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/4404fdefe1bf47a4ac66f4731ed862b2, entries=150, sequenceid=157, filesize=11.9 K 2024-11-18T20:43:08,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/fd1a71fcde674caaa55a35b2644d78df as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/fd1a71fcde674caaa55a35b2644d78df 2024-11-18T20:43:08,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/fd1a71fcde674caaa55a35b2644d78df, entries=150, sequenceid=157, filesize=11.9 K 2024-11-18T20:43:08,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/9a352d45fa034f62bce432b77b111549 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/9a352d45fa034f62bce432b77b111549 2024-11-18T20:43:08,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/9a352d45fa034f62bce432b77b111549, entries=150, sequenceid=157, filesize=11.9 K 2024-11-18T20:43:08,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 22cccb73b6312f1ad45031615ae4847f in 1244ms, sequenceid=157, compaction requested=false 2024-11-18T20:43:08,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:08,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:08,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-18T20:43:08,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:08,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:08,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:08,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:08,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:08,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:08,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/f7b546633ae04eb9bd35a7d7dd9a1e67 is 50, key is test_row_0/A:col10/1731962588795/Put/seqid=0 2024-11-18T20:43:08,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742287_1463 (size=14541) 2024-11-18T20:43:08,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:43:08,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:08,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:08,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:08,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:08,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:08,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962648829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:08,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:08,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962648830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:08,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:08,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962648835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:08,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:08,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962648947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:08,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962648947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:08,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962648957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:08,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:43:08,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:08,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:08,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:08,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:08,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:43:09,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:43:09,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:09,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:09,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:09,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:09,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:09,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962649167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:09,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962649167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:09,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962649167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:09,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/f7b546633ae04eb9bd35a7d7dd9a1e67 2024-11-18T20:43:09,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/86b974fa75a744b28b0546f180df43f9 is 50, key is test_row_0/B:col10/1731962588795/Put/seqid=0 2024-11-18T20:43:09,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742288_1464 (size=12151) 2024-11-18T20:43:09,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/86b974fa75a744b28b0546f180df43f9 2024-11-18T20:43:09,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/bec00e035e01410caa039192cfd9d7d5 is 50, key is test_row_0/C:col10/1731962588795/Put/seqid=0 2024-11-18T20:43:09,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742289_1465 (size=12151) 2024-11-18T20:43:09,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/bec00e035e01410caa039192cfd9d7d5 2024-11-18T20:43:09,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/f7b546633ae04eb9bd35a7d7dd9a1e67 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/f7b546633ae04eb9bd35a7d7dd9a1e67 2024-11-18T20:43:09,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/f7b546633ae04eb9bd35a7d7dd9a1e67, entries=200, sequenceid=195, filesize=14.2 K 2024-11-18T20:43:09,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/86b974fa75a744b28b0546f180df43f9 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/86b974fa75a744b28b0546f180df43f9 2024-11-18T20:43:09,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/86b974fa75a744b28b0546f180df43f9, entries=150, sequenceid=195, filesize=11.9 K 2024-11-18T20:43:09,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/bec00e035e01410caa039192cfd9d7d5 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/bec00e035e01410caa039192cfd9d7d5 2024-11-18T20:43:09,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/bec00e035e01410caa039192cfd9d7d5, entries=150, sequenceid=195, filesize=11.9 K 2024-11-18T20:43:09,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 22cccb73b6312f1ad45031615ae4847f in 443ms, sequenceid=195, compaction requested=true 2024-11-18T20:43:09,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:09,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:43:09,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:09,241 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:09,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:43:09,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:09,241 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:09,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:43:09,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:09,242 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:09,242 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:09,242 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/B is initiating minor compaction (all files) 2024-11-18T20:43:09,242 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/A is initiating minor compaction (all files) 2024-11-18T20:43:09,242 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/A in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:09,242 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/B in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:09,242 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/eeb2ec43f6a14bc58381b3b67db0cbb9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/fd1a71fcde674caaa55a35b2644d78df, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/86b974fa75a744b28b0546f180df43f9] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=35.8 K 2024-11-18T20:43:09,242 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/1bdf7aff363a4425900239cfe31a3770, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/4404fdefe1bf47a4ac66f4731ed862b2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/f7b546633ae04eb9bd35a7d7dd9a1e67] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=38.2 K 2024-11-18T20:43:09,242 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting eeb2ec43f6a14bc58381b3b67db0cbb9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731962586491 2024-11-18T20:43:09,242 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bdf7aff363a4425900239cfe31a3770, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731962586491 2024-11-18T20:43:09,242 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting fd1a71fcde674caaa55a35b2644d78df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731962586832 2024-11-18T20:43:09,242 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4404fdefe1bf47a4ac66f4731ed862b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731962586832 2024-11-18T20:43:09,242 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 86b974fa75a744b28b0546f180df43f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731962587593 2024-11-18T20:43:09,243 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7b546633ae04eb9bd35a7d7dd9a1e67, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731962587593 2024-11-18T20:43:09,249 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#B#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:09,249 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#A#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:09,249 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/40a6f47105734bf5915391a4800321cc is 50, key is test_row_0/B:col10/1731962588795/Put/seqid=0 2024-11-18T20:43:09,249 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/0ba14a996aa34273940b5d32f1ba7d1b is 50, key is test_row_0/A:col10/1731962588795/Put/seqid=0 2024-11-18T20:43:09,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742291_1467 (size=12493) 2024-11-18T20:43:09,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742290_1466 (size=12493) 2024-11-18T20:43:09,258 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/0ba14a996aa34273940b5d32f1ba7d1b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/0ba14a996aa34273940b5d32f1ba7d1b 2024-11-18T20:43:09,263 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/40a6f47105734bf5915391a4800321cc as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/40a6f47105734bf5915391a4800321cc 2024-11-18T20:43:09,263 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/A of 22cccb73b6312f1ad45031615ae4847f into 0ba14a996aa34273940b5d32f1ba7d1b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:09,263 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:09,263 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/A, priority=13, startTime=1731962589241; duration=0sec 2024-11-18T20:43:09,263 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:09,263 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:A 2024-11-18T20:43:09,263 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:09,263 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:09,264 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/C is initiating minor compaction (all files) 2024-11-18T20:43:09,264 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/C in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:09,264 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/2596a6a0a0994a1da21f10ac6d12f1b2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/9a352d45fa034f62bce432b77b111549, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/bec00e035e01410caa039192cfd9d7d5] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=35.8 K 2024-11-18T20:43:09,264 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2596a6a0a0994a1da21f10ac6d12f1b2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731962586491 2024-11-18T20:43:09,264 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a352d45fa034f62bce432b77b111549, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731962586832 2024-11-18T20:43:09,264 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting bec00e035e01410caa039192cfd9d7d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731962587593 2024-11-18T20:43:09,268 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/B of 22cccb73b6312f1ad45031615ae4847f into 40a6f47105734bf5915391a4800321cc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:09,268 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:09,268 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/B, priority=13, startTime=1731962589241; duration=0sec 2024-11-18T20:43:09,268 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:09,268 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:B 2024-11-18T20:43:09,272 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#C#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:09,272 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/528e78ea53a142b7a5e7da9d56e8a4ae is 50, key is test_row_0/C:col10/1731962588795/Put/seqid=0 2024-11-18T20:43:09,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742292_1468 (size=12493) 2024-11-18T20:43:09,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:43:09,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:09,297 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-18T20:43:09,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:09,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:09,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:09,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:09,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:09,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:09,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/52bfa360a39b4ee583e2034015eb88bf is 50, key is test_row_0/A:col10/1731962588820/Put/seqid=0 2024-11-18T20:43:09,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742293_1469 (size=12151) 2024-11-18T20:43:09,319 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/52bfa360a39b4ee583e2034015eb88bf 2024-11-18T20:43:09,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/5fa75ed40e37473084fbcb5463c4bff3 is 50, key is test_row_0/B:col10/1731962588820/Put/seqid=0 2024-11-18T20:43:09,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742294_1470 (size=12151) 2024-11-18T20:43:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:09,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:43:09,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962589521, max timestamp=1731962589556Num uniques 859; 2024-11-18T20:43:09,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:43:09,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962589521, max timestamp=1731962589556Num uniques 859; 2024-11-18T20:43:09,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962589521, max timestamp=1731962589556Num uniques 859; 2024-11-18T20:43:09,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:43:09,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:43:09,682 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/528e78ea53a142b7a5e7da9d56e8a4ae as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/528e78ea53a142b7a5e7da9d56e8a4ae 2024-11-18T20:43:09,685 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/C of 22cccb73b6312f1ad45031615ae4847f into 528e78ea53a142b7a5e7da9d56e8a4ae(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:09,685 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:09,686 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/C, priority=13, startTime=1731962589241; duration=0sec 2024-11-18T20:43:09,686 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:09,686 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:C 2024-11-18T20:43:09,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962649675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:09,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:09,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962649675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:09,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:09,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:09,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:09,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:09,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:09,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962649684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:09,738 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/5fa75ed40e37473084fbcb5463c4bff3 2024-11-18T20:43:09,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/1d284a3efb6f4d91a60a8bc7a03574c7 is 50, key is test_row_0/C:col10/1731962588820/Put/seqid=0 2024-11-18T20:43:09,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742295_1471 (size=12151) 2024-11-18T20:43:09,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962649797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:09,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962649797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:09,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962649807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:09,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:09,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:09,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:09,825 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T20:43:09,825 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-18T20:43:10,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962650017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962650017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962650017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:10,146 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/1d284a3efb6f4d91a60a8bc7a03574c7 2024-11-18T20:43:10,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/52bfa360a39b4ee583e2034015eb88bf as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/52bfa360a39b4ee583e2034015eb88bf 2024-11-18T20:43:10,153 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/52bfa360a39b4ee583e2034015eb88bf, entries=150, sequenceid=206, filesize=11.9 K 2024-11-18T20:43:10,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/5fa75ed40e37473084fbcb5463c4bff3 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/5fa75ed40e37473084fbcb5463c4bff3 2024-11-18T20:43:10,158 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/5fa75ed40e37473084fbcb5463c4bff3, entries=150, sequenceid=206, filesize=11.9 K 2024-11-18T20:43:10,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/1d284a3efb6f4d91a60a8bc7a03574c7 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/1d284a3efb6f4d91a60a8bc7a03574c7 2024-11-18T20:43:10,166 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/1d284a3efb6f4d91a60a8bc7a03574c7, entries=150, sequenceid=206, filesize=11.9 K 2024-11-18T20:43:10,167 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 22cccb73b6312f1ad45031615ae4847f in 870ms, sequenceid=206, compaction requested=false 2024-11-18T20:43:10,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:10,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:10,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-18T20:43:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-18T20:43:10,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-18T20:43:10,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7890 sec 2024-11-18T20:43:10,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.7930 sec 2024-11-18T20:43:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:10,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=154.31 KB heapSize=440.77 KB 2024-11-18T20:43:10,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:10,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T20:43:10,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:10,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T20:43:10,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:10,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T20:43:10,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/cac31bfa3f8c4c3e9f8d6740a3d1f3c6 is 50, key is test_row_0/A:col10/1731962590347/Put/seqid=0 2024-11-18T20:43:10,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742296_1472 (size=19321) 2024-11-18T20:43:10,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962650358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:10,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:10,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962650362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:10,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962650363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:10,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962650467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:10,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962650488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962650488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:43:10,528 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:10,528 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:43:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-18T20:43:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T20:43:10,530 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:43:10,530 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:43:10,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:43:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T20:43:10,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T20:43:10,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:10,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:10,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:10,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:10,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:10,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:10,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:10,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962650687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:10,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962650707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:10,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:10,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962650707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:10,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:10,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:10,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:10,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962650727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:10,734 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 22cccb73b6312f1ad45031615ae4847f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:10,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962650737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:10,749 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in 22cccb73b6312f1ad45031615ae4847f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:10,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/cac31bfa3f8c4c3e9f8d6740a3d1f3c6 2024-11-18T20:43:10,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/dd5de677956b4e749e166c1f39bbc34f is 50, key is test_row_0/B:col10/1731962590347/Put/seqid=0 2024-11-18T20:43:10,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742297_1473 (size=12151) 2024-11-18T20:43:10,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T20:43:10,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:10,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:10,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:10,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:10,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T20:43:10,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T20:43:10,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:10,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:10,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:10,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:10,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:11,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962651007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:11,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962651027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:11,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962651028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:11,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T20:43:11,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:11,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:11,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:11,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:11,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T20:43:11,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/dd5de677956b4e749e166c1f39bbc34f 2024-11-18T20:43:11,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/f8d03d6a0b8842458a4a43316ba63031 is 50, key is test_row_0/C:col10/1731962590347/Put/seqid=0 2024-11-18T20:43:11,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742298_1474 (size=12151) 2024-11-18T20:43:11,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T20:43:11,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:11,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:11,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:11,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:11,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:11,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T20:43:11,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:11,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:11,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:11,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:11,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:11,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962651517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:11,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:11,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:11,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962651557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:11,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:11,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:11,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962651557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:11,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:11,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:11,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:11,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:11,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/f8d03d6a0b8842458a4a43316ba63031 2024-11-18T20:43:11,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/cac31bfa3f8c4c3e9f8d6740a3d1f3c6 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/cac31bfa3f8c4c3e9f8d6740a3d1f3c6 2024-11-18T20:43:11,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/cac31bfa3f8c4c3e9f8d6740a3d1f3c6, entries=300, sequenceid=247, filesize=18.9 K 2024-11-18T20:43:11,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/dd5de677956b4e749e166c1f39bbc34f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/dd5de677956b4e749e166c1f39bbc34f 2024-11-18T20:43:11,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/dd5de677956b4e749e166c1f39bbc34f, entries=150, sequenceid=247, filesize=11.9 K 2024-11-18T20:43:11,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/f8d03d6a0b8842458a4a43316ba63031 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/f8d03d6a0b8842458a4a43316ba63031 2024-11-18T20:43:11,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/f8d03d6a0b8842458a4a43316ba63031, entries=150, sequenceid=247, filesize=11.9 K 2024-11-18T20:43:11,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~460.64 KB/471696, currentSize=20.13 KB/20610 for 22cccb73b6312f1ad45031615ae4847f in 1247ms, sequenceid=247, compaction requested=true 2024-11-18T20:43:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:43:11,594 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:11,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:43:11,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:11,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:43:11,595 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:11,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:11,595 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:11,595 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:11,595 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/B is initiating minor compaction (all files) 2024-11-18T20:43:11,595 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/A is initiating minor compaction (all files) 2024-11-18T20:43:11,595 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/A in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:11,595 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/B in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:11,595 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/0ba14a996aa34273940b5d32f1ba7d1b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/52bfa360a39b4ee583e2034015eb88bf, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/cac31bfa3f8c4c3e9f8d6740a3d1f3c6] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=42.9 K 2024-11-18T20:43:11,595 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/40a6f47105734bf5915391a4800321cc, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/5fa75ed40e37473084fbcb5463c4bff3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/dd5de677956b4e749e166c1f39bbc34f] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=35.9 K 2024-11-18T20:43:11,596 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 40a6f47105734bf5915391a4800321cc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731962587593 2024-11-18T20:43:11,596 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ba14a996aa34273940b5d32f1ba7d1b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731962587593 2024-11-18T20:43:11,596 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52bfa360a39b4ee583e2034015eb88bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731962588820 2024-11-18T20:43:11,596 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fa75ed40e37473084fbcb5463c4bff3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731962588820 2024-11-18T20:43:11,596 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting cac31bfa3f8c4c3e9f8d6740a3d1f3c6, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731962589673 2024-11-18T20:43:11,596 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting dd5de677956b4e749e166c1f39bbc34f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731962589682 2024-11-18T20:43:11,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T20:43:11,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:11,601 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-18T20:43:11,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:11,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:11,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:11,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:11,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:11,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:11,603 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#B#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:11,603 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/6e21189157d0491da4dc296f7a327413 is 50, key is test_row_0/B:col10/1731962590347/Put/seqid=0 2024-11-18T20:43:11,604 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#A#compaction#405 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:11,604 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/fe7cbed39650404eb51cfc7a84aa10ff is 50, key is test_row_0/A:col10/1731962590347/Put/seqid=0 2024-11-18T20:43:11,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/8e4813976c6441538d1ceeec3002d8b8 is 50, key is test_row_1/A:col10/1731962590356/Put/seqid=0 2024-11-18T20:43:11,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742299_1475 (size=12595) 2024-11-18T20:43:11,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742301_1477 (size=9757) 2024-11-18T20:43:11,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742300_1476 (size=12595) 2024-11-18T20:43:11,616 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/6e21189157d0491da4dc296f7a327413 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/6e21189157d0491da4dc296f7a327413 2024-11-18T20:43:11,620 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/B of 22cccb73b6312f1ad45031615ae4847f into 6e21189157d0491da4dc296f7a327413(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:11,620 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:11,620 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/B, priority=13, startTime=1731962591594; duration=0sec 2024-11-18T20:43:11,620 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:11,620 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:B 2024-11-18T20:43:11,620 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:11,620 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:11,621 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/C is initiating minor compaction (all files) 2024-11-18T20:43:11,621 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/C in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:11,621 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/528e78ea53a142b7a5e7da9d56e8a4ae, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/1d284a3efb6f4d91a60a8bc7a03574c7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/f8d03d6a0b8842458a4a43316ba63031] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=35.9 K 2024-11-18T20:43:11,621 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 528e78ea53a142b7a5e7da9d56e8a4ae, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731962587593 2024-11-18T20:43:11,621 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d284a3efb6f4d91a60a8bc7a03574c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731962588820 2024-11-18T20:43:11,621 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting f8d03d6a0b8842458a4a43316ba63031, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731962589682 2024-11-18T20:43:11,627 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#C#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:11,627 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/b7e8c4b7e22b455e8f3faed8a8f09779 is 50, key is test_row_0/C:col10/1731962590347/Put/seqid=0 2024-11-18T20:43:11,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742302_1478 (size=12595) 2024-11-18T20:43:11,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T20:43:12,012 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/8e4813976c6441538d1ceeec3002d8b8 2024-11-18T20:43:12,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/e2d3edafc8044131840145a15139cb55 is 50, key is test_row_1/B:col10/1731962590356/Put/seqid=0 2024-11-18T20:43:12,023 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/fe7cbed39650404eb51cfc7a84aa10ff as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/fe7cbed39650404eb51cfc7a84aa10ff 2024-11-18T20:43:12,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742303_1479 (size=9757) 2024-11-18T20:43:12,028 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/e2d3edafc8044131840145a15139cb55 2024-11-18T20:43:12,030 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/A of 22cccb73b6312f1ad45031615ae4847f into fe7cbed39650404eb51cfc7a84aa10ff(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:12,030 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:12,030 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/A, priority=13, startTime=1731962591594; duration=0sec 2024-11-18T20:43:12,030 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:12,030 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:A 2024-11-18T20:43:12,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/5a2a575d556246c2b1c8e0d6b046b6b9 is 50, key is test_row_1/C:col10/1731962590356/Put/seqid=0 2024-11-18T20:43:12,041 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/b7e8c4b7e22b455e8f3faed8a8f09779 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/b7e8c4b7e22b455e8f3faed8a8f09779 2024-11-18T20:43:12,047 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/C of 22cccb73b6312f1ad45031615ae4847f into b7e8c4b7e22b455e8f3faed8a8f09779(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:12,047 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:12,048 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/C, priority=13, startTime=1731962591595; duration=0sec 2024-11-18T20:43:12,048 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:12,048 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:C 2024-11-18T20:43:12,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742304_1480 (size=9757) 2024-11-18T20:43:12,058 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/5a2a575d556246c2b1c8e0d6b046b6b9 2024-11-18T20:43:12,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/8e4813976c6441538d1ceeec3002d8b8 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/8e4813976c6441538d1ceeec3002d8b8 2024-11-18T20:43:12,069 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/8e4813976c6441538d1ceeec3002d8b8, entries=100, sequenceid=253, filesize=9.5 K 2024-11-18T20:43:12,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/e2d3edafc8044131840145a15139cb55 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/e2d3edafc8044131840145a15139cb55 2024-11-18T20:43:12,076 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/e2d3edafc8044131840145a15139cb55, entries=100, sequenceid=253, filesize=9.5 K 2024-11-18T20:43:12,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/5a2a575d556246c2b1c8e0d6b046b6b9 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/5a2a575d556246c2b1c8e0d6b046b6b9 2024-11-18T20:43:12,084 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/5a2a575d556246c2b1c8e0d6b046b6b9, entries=100, sequenceid=253, filesize=9.5 K 2024-11-18T20:43:12,084 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 22cccb73b6312f1ad45031615ae4847f in 483ms, sequenceid=253, compaction requested=false 2024-11-18T20:43:12,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:12,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:12,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-18T20:43:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-18T20:43:12,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-18T20:43:12,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5560 sec 2024-11-18T20:43:12,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.5610 sec 2024-11-18T20:43:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:12,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T20:43:12,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:12,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:12,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:12,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:12,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:12,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:12,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/aa1975ef84f24dbbaab34a7de7c5cda7 is 50, key is test_row_0/A:col10/1731962592544/Put/seqid=0 2024-11-18T20:43:12,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742305_1481 (size=14741) 2024-11-18T20:43:12,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962652642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:12,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962592619, max timestamp=1731962592642Num uniques 859; 2024-11-18T20:43:12,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:43:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:12,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962592619, max timestamp=1731962592642Num uniques 859; 2024-11-18T20:43:12,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:43:12,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731962592619, max timestamp=1731962592642Num uniques 859; 2024-11-18T20:43:12,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:43:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T20:43:12,681 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:12,682 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:43:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-18T20:43:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T20:43:12,683 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:43:12,684 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:43:12,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:43:12,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962652742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:12,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962652743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:12,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962652757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T20:43:12,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:43:12,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:12,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:12,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:12,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:12,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:12,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:12,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962652857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:12,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:12,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:12,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:12,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962652867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:12,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/aa1975ef84f24dbbaab34a7de7c5cda7 2024-11-18T20:43:12,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:12,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962652977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:12,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:12,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:43:12,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:12,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:12,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:12,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:12,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:12,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:12,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/e6a3c22a19574c9294987df6c4b98494 is 50, key is test_row_0/B:col10/1731962592544/Put/seqid=0 2024-11-18T20:43:12,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742306_1482 (size=12301) 2024-11-18T20:43:12,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T20:43:13,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:13,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962653077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:13,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:13,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:13,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962653088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:13,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:43:13,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:13,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:13,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:13,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:13,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:13,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:13,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:43:13,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:13,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:13,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:13,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:13,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:13,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T20:43:13,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962653298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:13,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/e6a3c22a19574c9294987df6c4b98494 2024-11-18T20:43:13,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/45ea6803b1cd4faf8b2eea997fedc6db is 50, key is test_row_0/C:col10/1731962592544/Put/seqid=0 2024-11-18T20:43:13,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962653397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:13,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:13,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:13,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:13,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742307_1483 (size=12301) 2024-11-18T20:43:13,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962653407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:13,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:43:13,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:13,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:13,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:13,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:13,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:13,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:13,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:43:13,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:13,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:13,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:13,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:13,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:13,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:43:13,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:13,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:13,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:13,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:13,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:13,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:13,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/45ea6803b1cd4faf8b2eea997fedc6db 2024-11-18T20:43:13,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/aa1975ef84f24dbbaab34a7de7c5cda7 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/aa1975ef84f24dbbaab34a7de7c5cda7 2024-11-18T20:43:13,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/aa1975ef84f24dbbaab34a7de7c5cda7, entries=200, sequenceid=266, filesize=14.4 K 2024-11-18T20:43:13,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T20:43:13,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/e6a3c22a19574c9294987df6c4b98494 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/e6a3c22a19574c9294987df6c4b98494 2024-11-18T20:43:13,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/e6a3c22a19574c9294987df6c4b98494, entries=150, sequenceid=266, filesize=12.0 K 2024-11-18T20:43:13,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/45ea6803b1cd4faf8b2eea997fedc6db as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/45ea6803b1cd4faf8b2eea997fedc6db 2024-11-18T20:43:13,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/45ea6803b1cd4faf8b2eea997fedc6db, entries=150, sequenceid=266, filesize=12.0 K 2024-11-18T20:43:13,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 22cccb73b6312f1ad45031615ae4847f in 1254ms, sequenceid=266, compaction requested=true 2024-11-18T20:43:13,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:13,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:43:13,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:13,827 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:13,827 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:13,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:43:13,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:13,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:43:13,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:13,828 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:13,828 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:13,828 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/B is initiating minor compaction (all files) 2024-11-18T20:43:13,828 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/A is initiating minor compaction (all files) 2024-11-18T20:43:13,828 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/B in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:13,828 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/A in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:13,828 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/6e21189157d0491da4dc296f7a327413, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/e2d3edafc8044131840145a15139cb55, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/e6a3c22a19574c9294987df6c4b98494] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=33.8 K 2024-11-18T20:43:13,828 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/fe7cbed39650404eb51cfc7a84aa10ff, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/8e4813976c6441538d1ceeec3002d8b8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/aa1975ef84f24dbbaab34a7de7c5cda7] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=36.2 K 2024-11-18T20:43:13,828 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e21189157d0491da4dc296f7a327413, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731962589682 2024-11-18T20:43:13,828 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe7cbed39650404eb51cfc7a84aa10ff, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731962589682 2024-11-18T20:43:13,828 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting e2d3edafc8044131840145a15139cb55, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731962590356 2024-11-18T20:43:13,828 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting e6a3c22a19574c9294987df6c4b98494, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731962592537 2024-11-18T20:43:13,829 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e4813976c6441538d1ceeec3002d8b8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731962590356 2024-11-18T20:43:13,829 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa1975ef84f24dbbaab34a7de7c5cda7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731962592537 2024-11-18T20:43:13,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:13,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-18T20:43:13,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:13,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:13,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:13,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:13,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:13,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:13,835 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#B#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:13,835 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/fc406c98daa7457e81a4c1c192058795 is 50, key is test_row_0/B:col10/1731962592544/Put/seqid=0 2024-11-18T20:43:13,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/69a54dc4528b4d00a015c3e9614b5b3a is 50, key is test_row_0/A:col10/1731962593831/Put/seqid=0 2024-11-18T20:43:13,838 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#A#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:13,839 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/b7ad49568fa540f7ae1bd2d55799dd19 is 50, key is test_row_0/A:col10/1731962592544/Put/seqid=0 2024-11-18T20:43:13,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742308_1484 (size=12847) 2024-11-18T20:43:13,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742310_1486 (size=12847) 2024-11-18T20:43:13,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742309_1485 (size=14741) 2024-11-18T20:43:13,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:13,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962653890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:13,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:43:13,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:13,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:13,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:13,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:13,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:13,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962653917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:13,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:13,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962653927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:14,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962654008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:14,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:14,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:14,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:14,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:43:14,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:14,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:14,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:14,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:43:14,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:14,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:14,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:14,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962654227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:14,250 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/fc406c98daa7457e81a4c1c192058795 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/fc406c98daa7457e81a4c1c192058795 2024-11-18T20:43:14,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/69a54dc4528b4d00a015c3e9614b5b3a 2024-11-18T20:43:14,254 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/B of 22cccb73b6312f1ad45031615ae4847f into fc406c98daa7457e81a4c1c192058795(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:14,254 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:14,254 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/B, priority=13, startTime=1731962593827; duration=0sec 2024-11-18T20:43:14,254 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:14,254 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:B 2024-11-18T20:43:14,255 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:14,256 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/b7ad49568fa540f7ae1bd2d55799dd19 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/b7ad49568fa540f7ae1bd2d55799dd19 2024-11-18T20:43:14,256 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:14,256 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/C is initiating minor compaction (all files) 2024-11-18T20:43:14,256 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/C in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:14,256 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/b7e8c4b7e22b455e8f3faed8a8f09779, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/5a2a575d556246c2b1c8e0d6b046b6b9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/45ea6803b1cd4faf8b2eea997fedc6db] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=33.8 K 2024-11-18T20:43:14,257 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting b7e8c4b7e22b455e8f3faed8a8f09779, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731962589682 2024-11-18T20:43:14,257 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a2a575d556246c2b1c8e0d6b046b6b9, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731962590356 2024-11-18T20:43:14,257 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 45ea6803b1cd4faf8b2eea997fedc6db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731962592537 2024-11-18T20:43:14,261 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/A of 22cccb73b6312f1ad45031615ae4847f into b7ad49568fa540f7ae1bd2d55799dd19(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:14,261 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:14,261 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/A, priority=13, startTime=1731962593827; duration=0sec 2024-11-18T20:43:14,261 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:14,261 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:A 2024-11-18T20:43:14,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/2ffde4ae01714b0c8a0fa1c37c6ab607 is 50, key is test_row_0/B:col10/1731962593831/Put/seqid=0 2024-11-18T20:43:14,266 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#C#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:14,266 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/a2ce6f495b9b4817989492ca978d688f is 50, key is test_row_0/C:col10/1731962592544/Put/seqid=0 2024-11-18T20:43:14,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742311_1487 (size=12301) 2024-11-18T20:43:14,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742312_1488 (size=12847) 2024-11-18T20:43:14,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:43:14,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:14,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:14,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:14,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:43:14,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:14,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:14,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:14,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962654547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:14,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/2ffde4ae01714b0c8a0fa1c37c6ab607 2024-11-18T20:43:14,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:43:14,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:14,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:14,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:14,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/5345d82ba035435c8f04371cf717c16a is 50, key is test_row_0/C:col10/1731962593831/Put/seqid=0 2024-11-18T20:43:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,677 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/a2ce6f495b9b4817989492ca978d688f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/a2ce6f495b9b4817989492ca978d688f 2024-11-18T20:43:14,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742313_1489 (size=12301) 2024-11-18T20:43:14,681 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/C of 22cccb73b6312f1ad45031615ae4847f into a2ce6f495b9b4817989492ca978d688f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:14,681 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:14,681 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/C, priority=13, startTime=1731962593827; duration=0sec 2024-11-18T20:43:14,681 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:14,681 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:C 2024-11-18T20:43:14,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58708 deadline: 1731962654757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:14,765 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in 22cccb73b6312f1ad45031615ae4847f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:14,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58666 deadline: 1731962654778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:14,785 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in 22cccb73b6312f1ad45031615ae4847f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8291 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T20:43:14,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:43:14,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:14,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:14,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:14,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962654927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:14,945 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in 22cccb73b6312f1ad45031615ae4847f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:14,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962654947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:14,959 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in 22cccb73b6312f1ad45031615ae4847f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:14,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:14,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:14,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:14,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:43:14,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:14,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:14,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:14,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:15,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962655067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:15,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:15,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:15,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:15,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/5345d82ba035435c8f04371cf717c16a 2024-11-18T20:43:15,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/69a54dc4528b4d00a015c3e9614b5b3a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/69a54dc4528b4d00a015c3e9614b5b3a 2024-11-18T20:43:15,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/69a54dc4528b4d00a015c3e9614b5b3a, entries=200, sequenceid=303, filesize=14.4 K 2024-11-18T20:43:15,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/2ffde4ae01714b0c8a0fa1c37c6ab607 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/2ffde4ae01714b0c8a0fa1c37c6ab607 2024-11-18T20:43:15,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/2ffde4ae01714b0c8a0fa1c37c6ab607, entries=150, sequenceid=303, filesize=12.0 K 2024-11-18T20:43:15,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/5345d82ba035435c8f04371cf717c16a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/5345d82ba035435c8f04371cf717c16a 2024-11-18T20:43:15,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/5345d82ba035435c8f04371cf717c16a, entries=150, sequenceid=303, filesize=12.0 K 2024-11-18T20:43:15,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 22cccb73b6312f1ad45031615ae4847f in 1265ms, sequenceid=303, compaction requested=false 2024-11-18T20:43:15,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:15,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:43:15,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:15,134 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-18T20:43:15,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:15,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:15,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:15,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:15,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:15,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:15,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/328d11db81b5419585bd7b927dc3c421 is 50, key is test_row_0/A:col10/1731962593889/Put/seqid=0 2024-11-18T20:43:15,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742314_1490 (size=12301) 2024-11-18T20:43:15,543 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/328d11db81b5419585bd7b927dc3c421 2024-11-18T20:43:15,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/ece7cb5b5375438299b72d18072270e9 is 50, key is test_row_0/B:col10/1731962593889/Put/seqid=0 2024-11-18T20:43:15,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742315_1491 (size=12301) 2024-11-18T20:43:15,553 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/ece7cb5b5375438299b72d18072270e9 2024-11-18T20:43:15,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/65f5f8c057fc4b3a93f5ce42b118d7df is 50, key is test_row_0/C:col10/1731962593889/Put/seqid=0 2024-11-18T20:43:15,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742316_1492 (size=12301) 2024-11-18T20:43:15,968 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/65f5f8c057fc4b3a93f5ce42b118d7df 2024-11-18T20:43:15,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/328d11db81b5419585bd7b927dc3c421 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/328d11db81b5419585bd7b927dc3c421 2024-11-18T20:43:15,976 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/328d11db81b5419585bd7b927dc3c421, entries=150, sequenceid=315, filesize=12.0 K 2024-11-18T20:43:15,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/ece7cb5b5375438299b72d18072270e9 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/ece7cb5b5375438299b72d18072270e9 2024-11-18T20:43:15,982 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/ece7cb5b5375438299b72d18072270e9, entries=150, sequenceid=315, filesize=12.0 K 2024-11-18T20:43:15,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/65f5f8c057fc4b3a93f5ce42b118d7df as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/65f5f8c057fc4b3a93f5ce42b118d7df 2024-11-18T20:43:15,988 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/65f5f8c057fc4b3a93f5ce42b118d7df, entries=150, sequenceid=315, filesize=12.0 K 2024-11-18T20:43:15,989 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 22cccb73b6312f1ad45031615ae4847f in 856ms, sequenceid=315, compaction requested=true 2024-11-18T20:43:15,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:15,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:15,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-18T20:43:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-18T20:43:15,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-18T20:43:15,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3070 sec 2024-11-18T20:43:15,995 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 3.3110 sec 2024-11-18T20:43:16,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:16,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T20:43:16,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:16,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:16,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:16,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:16,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:16,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:16,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/2046611bb5ce437fa5c1a39dd65bfc68 is 50, key is test_row_1/A:col10/1731962596106/Put/seqid=0 2024-11-18T20:43:16,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742317_1493 (size=14737) 2024-11-18T20:43:16,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962596242, max timestamp=1731962596292Num uniques 859; 2024-11-18T20:43:16,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:43:16,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962596242, max timestamp=1731962596292Num uniques 859; 2024-11-18T20:43:16,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:43:16,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962596242, max timestamp=1731962596292Num uniques 859; 2024-11-18T20:43:16,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:43:16,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962656501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:16,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/2046611bb5ce437fa5c1a39dd65bfc68 2024-11-18T20:43:16,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/23199c2c2f0242619dda153bdb04d72c is 50, key is test_row_1/B:col10/1731962596106/Put/seqid=0 2024-11-18T20:43:16,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742318_1494 (size=9857) 2024-11-18T20:43:16,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962656617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T20:43:16,838 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:16,838 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:43:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-18T20:43:16,840 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:43:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T20:43:16,841 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:43:16,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:43:16,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962656838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:16,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/23199c2c2f0242619dda153bdb04d72c 2024-11-18T20:43:16,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/fd1f6bb9f2bc48d3bf2f0b7e3a1c029f is 50, key is test_row_1/C:col10/1731962596106/Put/seqid=0 2024-11-18T20:43:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T20:43:16,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742319_1495 (size=9857) 2024-11-18T20:43:16,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962656957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:16,967 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in 22cccb73b6312f1ad45031615ae4847f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:16,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:16,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:16,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:16,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962656977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:16,988 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in 22cccb73b6312f1ad45031615ae4847f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:16,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:16,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T20:43:16,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:16,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:16,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:16,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:16,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:17,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T20:43:17,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:17,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:17,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:17,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:17,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T20:43:17,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:17,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962657158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:17,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:17,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:17,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:17,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T20:43:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:17,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:17,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/fd1f6bb9f2bc48d3bf2f0b7e3a1c029f 2024-11-18T20:43:17,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/2046611bb5ce437fa5c1a39dd65bfc68 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/2046611bb5ce437fa5c1a39dd65bfc68 2024-11-18T20:43:17,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/2046611bb5ce437fa5c1a39dd65bfc68, entries=200, sequenceid=325, filesize=14.4 K 2024-11-18T20:43:17,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/23199c2c2f0242619dda153bdb04d72c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/23199c2c2f0242619dda153bdb04d72c 2024-11-18T20:43:17,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/23199c2c2f0242619dda153bdb04d72c, entries=100, sequenceid=325, filesize=9.6 K 2024-11-18T20:43:17,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/fd1f6bb9f2bc48d3bf2f0b7e3a1c029f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/fd1f6bb9f2bc48d3bf2f0b7e3a1c029f 2024-11-18T20:43:17,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/fd1f6bb9f2bc48d3bf2f0b7e3a1c029f, entries=100, sequenceid=325, filesize=9.6 K 2024-11-18T20:43:17,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 22cccb73b6312f1ad45031615ae4847f in 1264ms, sequenceid=325, compaction requested=true 2024-11-18T20:43:17,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:17,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:43:17,379 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:43:17,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:17,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:43:17,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:17,379 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:43:17,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:43:17,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:17,380 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54626 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:43:17,380 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47306 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:43:17,380 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/B is initiating minor compaction (all files) 2024-11-18T20:43:17,380 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/A is initiating minor compaction (all files) 2024-11-18T20:43:17,380 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/A in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:17,380 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/B in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:17,380 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/b7ad49568fa540f7ae1bd2d55799dd19, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/69a54dc4528b4d00a015c3e9614b5b3a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/328d11db81b5419585bd7b927dc3c421, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/2046611bb5ce437fa5c1a39dd65bfc68] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=53.3 K 2024-11-18T20:43:17,380 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/fc406c98daa7457e81a4c1c192058795, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/2ffde4ae01714b0c8a0fa1c37c6ab607, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/ece7cb5b5375438299b72d18072270e9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/23199c2c2f0242619dda153bdb04d72c] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=46.2 K 2024-11-18T20:43:17,380 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7ad49568fa540f7ae1bd2d55799dd19, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731962592537 2024-11-18T20:43:17,380 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting fc406c98daa7457e81a4c1c192058795, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731962592537 2024-11-18T20:43:17,381 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69a54dc4528b4d00a015c3e9614b5b3a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731962592724 2024-11-18T20:43:17,381 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ffde4ae01714b0c8a0fa1c37c6ab607, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731962592725 2024-11-18T20:43:17,381 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 328d11db81b5419585bd7b927dc3c421, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731962593860 2024-11-18T20:43:17,381 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting ece7cb5b5375438299b72d18072270e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731962593860 2024-11-18T20:43:17,381 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2046611bb5ce437fa5c1a39dd65bfc68, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731962596099 2024-11-18T20:43:17,381 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 23199c2c2f0242619dda153bdb04d72c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731962596106 2024-11-18T20:43:17,387 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#A#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:17,387 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/23838230f37b4b01bc39410a4a6e3e3b is 50, key is test_row_0/A:col10/1731962593889/Put/seqid=0 2024-11-18T20:43:17,388 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#B#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:17,389 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/cb98cf04bcee41a181d13122522a27df is 50, key is test_row_0/B:col10/1731962593889/Put/seqid=0 2024-11-18T20:43:17,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742320_1496 (size=12983) 2024-11-18T20:43:17,400 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/23838230f37b4b01bc39410a4a6e3e3b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/23838230f37b4b01bc39410a4a6e3e3b 2024-11-18T20:43:17,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742321_1497 (size=12983) 2024-11-18T20:43:17,404 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/A of 22cccb73b6312f1ad45031615ae4847f into 23838230f37b4b01bc39410a4a6e3e3b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:17,404 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:17,405 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/A, priority=12, startTime=1731962597379; duration=0sec 2024-11-18T20:43:17,405 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:17,405 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:A 2024-11-18T20:43:17,405 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:43:17,406 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47306 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:43:17,406 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/C is initiating minor compaction (all files) 2024-11-18T20:43:17,406 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/C in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:17,406 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/a2ce6f495b9b4817989492ca978d688f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/5345d82ba035435c8f04371cf717c16a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/65f5f8c057fc4b3a93f5ce42b118d7df, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/fd1f6bb9f2bc48d3bf2f0b7e3a1c029f] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=46.2 K 2024-11-18T20:43:17,406 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2ce6f495b9b4817989492ca978d688f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731962592537 2024-11-18T20:43:17,406 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5345d82ba035435c8f04371cf717c16a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731962592725 2024-11-18T20:43:17,407 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65f5f8c057fc4b3a93f5ce42b118d7df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731962593860 2024-11-18T20:43:17,407 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd1f6bb9f2bc48d3bf2f0b7e3a1c029f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731962596106 2024-11-18T20:43:17,424 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#C#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:17,424 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/b3113399747b485eb4d154c80bb0bb1f is 50, key is test_row_0/C:col10/1731962593889/Put/seqid=0 2024-11-18T20:43:17,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742322_1498 (size=12983) 2024-11-18T20:43:17,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T20:43:17,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:17,452 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-18T20:43:17,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:17,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:17,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:17,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:17,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:17,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:17,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/acdea4c8325c40c1af6ec38d94de5329 is 50, key is test_row_0/A:col10/1731962596459/Put/seqid=0 2024-11-18T20:43:17,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742323_1499 (size=12301) 2024-11-18T20:43:17,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T20:43:17,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:17,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:17,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962657725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:17,805 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/cb98cf04bcee41a181d13122522a27df as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/cb98cf04bcee41a181d13122522a27df 2024-11-18T20:43:17,808 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/B of 22cccb73b6312f1ad45031615ae4847f into cb98cf04bcee41a181d13122522a27df(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:17,808 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:17,808 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/B, priority=12, startTime=1731962597379; duration=0sec 2024-11-18T20:43:17,808 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:17,808 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:B 2024-11-18T20:43:17,840 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/b3113399747b485eb4d154c80bb0bb1f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/b3113399747b485eb4d154c80bb0bb1f 2024-11-18T20:43:17,844 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/C of 22cccb73b6312f1ad45031615ae4847f into b3113399747b485eb4d154c80bb0bb1f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:17,844 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:17,844 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/C, priority=12, startTime=1731962597379; duration=0sec 2024-11-18T20:43:17,844 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:17,844 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:C 2024-11-18T20:43:17,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962657847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:17,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:17,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:17,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:17,862 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/acdea4c8325c40c1af6ec38d94de5329 2024-11-18T20:43:17,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/1af56fc0034243f2856f9ba7c88eef52 is 50, key is test_row_0/B:col10/1731962596459/Put/seqid=0 2024-11-18T20:43:17,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742324_1500 (size=12301) 2024-11-18T20:43:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T20:43:18,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962658067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:18,274 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/1af56fc0034243f2856f9ba7c88eef52 2024-11-18T20:43:18,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/7623bd36d6334aa28617199a21dc427a is 50, key is test_row_0/C:col10/1731962596459/Put/seqid=0 2024-11-18T20:43:18,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742325_1501 (size=12301) 2024-11-18T20:43:18,289 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/7623bd36d6334aa28617199a21dc427a 2024-11-18T20:43:18,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/acdea4c8325c40c1af6ec38d94de5329 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/acdea4c8325c40c1af6ec38d94de5329 2024-11-18T20:43:18,295 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/acdea4c8325c40c1af6ec38d94de5329, entries=150, sequenceid=363, filesize=12.0 K 2024-11-18T20:43:18,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/1af56fc0034243f2856f9ba7c88eef52 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/1af56fc0034243f2856f9ba7c88eef52 2024-11-18T20:43:18,302 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/1af56fc0034243f2856f9ba7c88eef52, entries=150, sequenceid=363, filesize=12.0 K 2024-11-18T20:43:18,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/7623bd36d6334aa28617199a21dc427a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/7623bd36d6334aa28617199a21dc427a 2024-11-18T20:43:18,307 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/7623bd36d6334aa28617199a21dc427a, entries=150, sequenceid=363, filesize=12.0 K 2024-11-18T20:43:18,310 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 22cccb73b6312f1ad45031615ae4847f in 858ms, sequenceid=363, compaction requested=false 2024-11-18T20:43:18,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:18,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:18,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-18T20:43:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-18T20:43:18,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-18T20:43:18,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4700 sec 2024-11-18T20:43:18,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.4760 sec 2024-11-18T20:43:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:18,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T20:43:18,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:18,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:18,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:18,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:18,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:18,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:18,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/f197682361c04d538baf3c04ef34980e is 50, key is test_row_0/A:col10/1731962597723/Put/seqid=0 2024-11-18T20:43:18,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742326_1502 (size=14741) 2024-11-18T20:43:18,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/f197682361c04d538baf3c04ef34980e 2024-11-18T20:43:18,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/09ea23f95e8a45ceb0dd6c0d04b5ff70 is 50, key is test_row_0/B:col10/1731962597723/Put/seqid=0 2024-11-18T20:43:18,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742327_1503 (size=12301) 2024-11-18T20:43:18,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962598490, max timestamp=1731962598607Num uniques 859; 2024-11-18T20:43:18,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962598490, max timestamp=1731962598607Num uniques 859; 2024-11-18T20:43:18,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:43:18,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:43:18,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731962598490, max timestamp=1731962598607Num uniques 859; 2024-11-18T20:43:18,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T20:43:18,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962658773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:18,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:18,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:18,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:18,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/09ea23f95e8a45ceb0dd6c0d04b5ff70 2024-11-18T20:43:18,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/2cfd9b0085af4a60a0fc8f3cc75a7664 is 50, key is test_row_0/C:col10/1731962597723/Put/seqid=0 2024-11-18T20:43:18,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742328_1504 (size=12301) 2024-11-18T20:43:18,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962658898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T20:43:18,987 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:18,988 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:43:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-18T20:43:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T20:43:18,990 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:43:18,990 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:43:18,990 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:43:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T20:43:19,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962659117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:19,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:19,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:19,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:19,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T20:43:19,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:19,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:19,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:19,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:19,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:19,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/2cfd9b0085af4a60a0fc8f3cc75a7664 2024-11-18T20:43:19,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/f197682361c04d538baf3c04ef34980e as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/f197682361c04d538baf3c04ef34980e 2024-11-18T20:43:19,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/f197682361c04d538baf3c04ef34980e, entries=200, sequenceid=376, filesize=14.4 K 2024-11-18T20:43:19,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/09ea23f95e8a45ceb0dd6c0d04b5ff70 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/09ea23f95e8a45ceb0dd6c0d04b5ff70 2024-11-18T20:43:19,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/09ea23f95e8a45ceb0dd6c0d04b5ff70, entries=150, sequenceid=376, filesize=12.0 K 2024-11-18T20:43:19,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/2cfd9b0085af4a60a0fc8f3cc75a7664 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/2cfd9b0085af4a60a0fc8f3cc75a7664 2024-11-18T20:43:19,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/2cfd9b0085af4a60a0fc8f3cc75a7664, entries=150, sequenceid=376, filesize=12.0 K 2024-11-18T20:43:19,242 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-18T20:43:19,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 22cccb73b6312f1ad45031615ae4847f in 850ms, sequenceid=376, compaction requested=true 2024-11-18T20:43:19,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:19,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:43:19,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:19,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:43:19,243 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:19,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:19,243 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:19,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:43:19,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:19,244 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:19,244 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:19,244 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/B is initiating minor compaction (all files) 2024-11-18T20:43:19,244 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/A is initiating minor compaction (all files) 2024-11-18T20:43:19,244 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/B in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:19,244 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/A in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:19,244 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/cb98cf04bcee41a181d13122522a27df, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/1af56fc0034243f2856f9ba7c88eef52, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/09ea23f95e8a45ceb0dd6c0d04b5ff70] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=36.7 K 2024-11-18T20:43:19,244 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/23838230f37b4b01bc39410a4a6e3e3b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/acdea4c8325c40c1af6ec38d94de5329, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/f197682361c04d538baf3c04ef34980e] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=39.1 K 2024-11-18T20:43:19,244 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting cb98cf04bcee41a181d13122522a27df, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731962593889 2024-11-18T20:43:19,244 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23838230f37b4b01bc39410a4a6e3e3b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731962593889 2024-11-18T20:43:19,245 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 1af56fc0034243f2856f9ba7c88eef52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731962596440 2024-11-18T20:43:19,245 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting acdea4c8325c40c1af6ec38d94de5329, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731962596440 2024-11-18T20:43:19,245 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 09ea23f95e8a45ceb0dd6c0d04b5ff70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731962597720 2024-11-18T20:43:19,245 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting f197682361c04d538baf3c04ef34980e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731962597677 2024-11-18T20:43:19,250 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#B#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:19,251 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#A#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:19,251 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/94c9929f422e4e52ab7693ff9e31e3ad is 50, key is test_row_0/B:col10/1731962597723/Put/seqid=0 2024-11-18T20:43:19,251 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/04d2447e1c624e7683aacb409e19731a is 50, key is test_row_0/A:col10/1731962597723/Put/seqid=0 2024-11-18T20:43:19,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742329_1505 (size=13085) 2024-11-18T20:43:19,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742330_1506 (size=13085) 2024-11-18T20:43:19,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T20:43:19,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:19,296 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-18T20:43:19,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:19,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:19,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:19,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:19,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:19,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:19,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/9e2b903ce67b4a809fb40afa5ed51e30 is 50, key is test_row_0/A:col10/1731962598754/Put/seqid=0 2024-11-18T20:43:19,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742331_1507 (size=12301) 2024-11-18T20:43:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T20:43:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:19,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962659533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:19,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T20:43:19,670 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/94c9929f422e4e52ab7693ff9e31e3ad as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/94c9929f422e4e52ab7693ff9e31e3ad 2024-11-18T20:43:19,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962659657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:19,674 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/04d2447e1c624e7683aacb409e19731a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/04d2447e1c624e7683aacb409e19731a 2024-11-18T20:43:19,678 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/B of 22cccb73b6312f1ad45031615ae4847f into 94c9929f422e4e52ab7693ff9e31e3ad(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:19,678 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:19,678 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/B, priority=13, startTime=1731962599243; duration=0sec 2024-11-18T20:43:19,678 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:19,678 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:B 2024-11-18T20:43:19,680 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:19,681 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:19,681 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/A of 22cccb73b6312f1ad45031615ae4847f into 04d2447e1c624e7683aacb409e19731a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:19,681 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/C is initiating minor compaction (all files) 2024-11-18T20:43:19,681 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:19,681 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/C in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:19,681 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/A, priority=13, startTime=1731962599243; duration=0sec 2024-11-18T20:43:19,681 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/b3113399747b485eb4d154c80bb0bb1f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/7623bd36d6334aa28617199a21dc427a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/2cfd9b0085af4a60a0fc8f3cc75a7664] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=36.7 K 2024-11-18T20:43:19,681 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:19,681 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:A 2024-11-18T20:43:19,682 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting b3113399747b485eb4d154c80bb0bb1f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731962593889 2024-11-18T20:43:19,682 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 7623bd36d6334aa28617199a21dc427a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731962596440 2024-11-18T20:43:19,683 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cfd9b0085af4a60a0fc8f3cc75a7664, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731962597720 2024-11-18T20:43:19,690 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#C#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:19,690 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/1d955d967dac4382a317aac9192d9886 is 50, key is test_row_0/C:col10/1731962597723/Put/seqid=0 2024-11-18T20:43:19,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742332_1508 (size=13085) 2024-11-18T20:43:19,703 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/9e2b903ce67b4a809fb40afa5ed51e30 2024-11-18T20:43:19,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/7d2e749cd35a4ffbb75e730a9bd619a4 is 50, key is test_row_0/B:col10/1731962598754/Put/seqid=0 2024-11-18T20:43:19,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742333_1509 (size=12301) 2024-11-18T20:43:19,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962659877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:20,100 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/1d955d967dac4382a317aac9192d9886 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/1d955d967dac4382a317aac9192d9886 2024-11-18T20:43:20,104 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/C of 22cccb73b6312f1ad45031615ae4847f into 1d955d967dac4382a317aac9192d9886(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:20,104 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:20,104 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/C, priority=13, startTime=1731962599243; duration=0sec 2024-11-18T20:43:20,104 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:20,104 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:C 2024-11-18T20:43:20,115 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/7d2e749cd35a4ffbb75e730a9bd619a4 2024-11-18T20:43:20,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/2f19a2b4b3db40aaba6e57f33dbea41a is 50, key is test_row_0/C:col10/1731962598754/Put/seqid=0 2024-11-18T20:43:20,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742334_1510 (size=12301) 2024-11-18T20:43:20,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T20:43:20,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962660187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:20,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:20,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:20,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:20,525 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/2f19a2b4b3db40aaba6e57f33dbea41a 2024-11-18T20:43:20,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/9e2b903ce67b4a809fb40afa5ed51e30 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/9e2b903ce67b4a809fb40afa5ed51e30 2024-11-18T20:43:20,533 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/9e2b903ce67b4a809fb40afa5ed51e30, entries=150, sequenceid=412, filesize=12.0 K 2024-11-18T20:43:20,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/7d2e749cd35a4ffbb75e730a9bd619a4 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/7d2e749cd35a4ffbb75e730a9bd619a4 2024-11-18T20:43:20,536 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/7d2e749cd35a4ffbb75e730a9bd619a4, entries=150, sequenceid=412, filesize=12.0 K 2024-11-18T20:43:20,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/2f19a2b4b3db40aaba6e57f33dbea41a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/2f19a2b4b3db40aaba6e57f33dbea41a 2024-11-18T20:43:20,539 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/2f19a2b4b3db40aaba6e57f33dbea41a, entries=150, sequenceid=412, filesize=12.0 K 2024-11-18T20:43:20,540 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 22cccb73b6312f1ad45031615ae4847f in 1244ms, sequenceid=412, compaction requested=false 2024-11-18T20:43:20,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:20,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:20,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-18T20:43:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-18T20:43:20,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-18T20:43:20,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5510 sec 2024-11-18T20:43:20,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.5560 sec 2024-11-18T20:43:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:20,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T20:43:20,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:20,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:20,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:20,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:20,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:20,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:20,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/28b86dff88a849968d324fb6d903710d is 50, key is test_row_0/A:col10/1731962599516/Put/seqid=0 2024-11-18T20:43:20,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742335_1511 (size=14741) 2024-11-18T20:43:20,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962660911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:20,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:20,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:20,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:21,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58652 deadline: 1731962660999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:21,012 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in 22cccb73b6312f1ad45031615ae4847f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8269 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:21,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:21,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:21,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:21,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58664 deadline: 1731962661008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:21,014 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in 22cccb73b6312f1ad45031615ae4847f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:21,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962661027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:21,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:21,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:21,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:21,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/28b86dff88a849968d324fb6d903710d 2024-11-18T20:43:21,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/ef9543c73b8145bbbf3fafb2d75067d9 is 50, key is test_row_0/B:col10/1731962599516/Put/seqid=0 2024-11-18T20:43:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T20:43:21,137 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:21,138 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:43:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-18T20:43:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:43:21,141 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:43:21,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742336_1512 (size=12301) 2024-11-18T20:43:21,143 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:43:21,143 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:43:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:43:21,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:21,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962661247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:21,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:21,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:21,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:21,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-18T20:43:21,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:21,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:21,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:21,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:21,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:21,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:21,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-18T20:43:21,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:21,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:21,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:21,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:21,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:43:21,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/ef9543c73b8145bbbf3fafb2d75067d9 2024-11-18T20:43:21,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/c6ecc9217d9e447690282630cda7c8f6 is 50, key is test_row_0/C:col10/1731962599516/Put/seqid=0 2024-11-18T20:43:21,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742337_1513 (size=12301) 2024-11-18T20:43:21,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/c6ecc9217d9e447690282630cda7c8f6 2024-11-18T20:43:21,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/28b86dff88a849968d324fb6d903710d as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/28b86dff88a849968d324fb6d903710d 2024-11-18T20:43:21,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/28b86dff88a849968d324fb6d903710d, entries=200, sequenceid=427, filesize=14.4 K 2024-11-18T20:43:21,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/ef9543c73b8145bbbf3fafb2d75067d9 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/ef9543c73b8145bbbf3fafb2d75067d9 2024-11-18T20:43:21,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58676 deadline: 1731962661567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 , the old value is region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., hostname=c0a89b2656d4,45393,1731962440107, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:43:21,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/ef9543c73b8145bbbf3fafb2d75067d9, entries=150, sequenceid=427, filesize=12.0 K 2024-11-18T20:43:21,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/c6ecc9217d9e447690282630cda7c8f6 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/c6ecc9217d9e447690282630cda7c8f6 2024-11-18T20:43:21,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/c6ecc9217d9e447690282630cda7c8f6, entries=150, sequenceid=427, filesize=12.0 K 2024-11-18T20:43:21,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 22cccb73b6312f1ad45031615ae4847f in 877ms, sequenceid=427, compaction requested=true 2024-11-18T20:43:21,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:21,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:43:21,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:21,586 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:21,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:43:21,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:21,586 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:21,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22cccb73b6312f1ad45031615ae4847f:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:43:21,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:21,587 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:21,587 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:21,587 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/A is initiating minor compaction (all files) 2024-11-18T20:43:21,587 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/B is initiating minor compaction (all files) 2024-11-18T20:43:21,587 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/A in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:21,587 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/B in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:21,587 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/94c9929f422e4e52ab7693ff9e31e3ad, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/7d2e749cd35a4ffbb75e730a9bd619a4, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/ef9543c73b8145bbbf3fafb2d75067d9] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=36.8 K 2024-11-18T20:43:21,587 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/04d2447e1c624e7683aacb409e19731a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/9e2b903ce67b4a809fb40afa5ed51e30, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/28b86dff88a849968d324fb6d903710d] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=39.2 K 2024-11-18T20:43:21,587 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 94c9929f422e4e52ab7693ff9e31e3ad, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731962597720 2024-11-18T20:43:21,588 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d2e749cd35a4ffbb75e730a9bd619a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731962598722 2024-11-18T20:43:21,588 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04d2447e1c624e7683aacb409e19731a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731962597720 2024-11-18T20:43:21,588 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting ef9543c73b8145bbbf3fafb2d75067d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731962599460 2024-11-18T20:43:21,588 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e2b903ce67b4a809fb40afa5ed51e30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731962598722 2024-11-18T20:43:21,588 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28b86dff88a849968d324fb6d903710d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731962599460 2024-11-18T20:43:21,595 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#B#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:21,596 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/2c5ddfb5cc874fbc86f4e4c08144a90a is 50, key is test_row_0/B:col10/1731962599516/Put/seqid=0 2024-11-18T20:43:21,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-18T20:43:21,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:21,601 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T20:43:21,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:21,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:21,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:21,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:21,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:21,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:21,607 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#A#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:21,608 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/8a6e2094434e4044994a49fb859dd9f9 is 50, key is test_row_0/A:col10/1731962599516/Put/seqid=0 2024-11-18T20:43:21,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/7e0406ce5ee24aaba212033d85f0a7ba is 50, key is test_row_0/A:col10/1731962600859/Put/seqid=0 2024-11-18T20:43:21,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742338_1514 (size=13187) 2024-11-18T20:43:21,647 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/2c5ddfb5cc874fbc86f4e4c08144a90a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/2c5ddfb5cc874fbc86f4e4c08144a90a 2024-11-18T20:43:21,652 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/B of 22cccb73b6312f1ad45031615ae4847f into 2c5ddfb5cc874fbc86f4e4c08144a90a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:21,652 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:21,653 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/B, priority=13, startTime=1731962601586; duration=0sec 2024-11-18T20:43:21,653 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:21,653 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:B 2024-11-18T20:43:21,653 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:21,654 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:21,654 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): 22cccb73b6312f1ad45031615ae4847f/C is initiating minor compaction (all files) 2024-11-18T20:43:21,654 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22cccb73b6312f1ad45031615ae4847f/C in TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:21,654 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/1d955d967dac4382a317aac9192d9886, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/2f19a2b4b3db40aaba6e57f33dbea41a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/c6ecc9217d9e447690282630cda7c8f6] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp, totalSize=36.8 K 2024-11-18T20:43:21,655 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d955d967dac4382a317aac9192d9886, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731962597720 2024-11-18T20:43:21,655 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f19a2b4b3db40aaba6e57f33dbea41a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731962598722 2024-11-18T20:43:21,655 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting c6ecc9217d9e447690282630cda7c8f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731962599460 2024-11-18T20:43:21,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742339_1515 (size=13187) 2024-11-18T20:43:21,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742340_1516 (size=12301) 2024-11-18T20:43:21,663 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22cccb73b6312f1ad45031615ae4847f#C#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:21,663 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/88717b0471e345f0b2019b6111130ecb is 50, key is test_row_0/C:col10/1731962599516/Put/seqid=0 2024-11-18T20:43:21,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742341_1517 (size=13187) 2024-11-18T20:43:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:43:22,059 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/8a6e2094434e4044994a49fb859dd9f9 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/8a6e2094434e4044994a49fb859dd9f9 2024-11-18T20:43:22,060 INFO [Thread-1982 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1982. 2024-11-18T20:43:22,061 DEBUG [Thread-1982 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:43:22,061 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:22,061 INFO [Thread-1978 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1978. 2024-11-18T20:43:22,061 DEBUG [Thread-1978 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:43:22,061 DEBUG [Thread-1978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:22,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:22,061 DEBUG [Thread-1978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:22,062 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/7e0406ce5ee24aaba212033d85f0a7ba 2024-11-18T20:43:22,062 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:22,062 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:22,064 INFO [Thread-1980 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1980. 2024-11-18T20:43:22,064 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/A of 22cccb73b6312f1ad45031615ae4847f into 8a6e2094434e4044994a49fb859dd9f9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:22,064 DEBUG [Thread-1980 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:43:22,064 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:22,064 DEBUG [Thread-1980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:22,064 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/A, priority=13, startTime=1731962601585; duration=0sec 2024-11-18T20:43:22,064 DEBUG [Thread-1980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:22,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:22,064 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:22,064 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:A 2024-11-18T20:43:22,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/3cade95168564923bbf5ba385a08cfd2 is 50, key is test_row_0/B:col10/1731962600859/Put/seqid=0 2024-11-18T20:43:22,069 INFO [Thread-1974 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1974. 2024-11-18T20:43:22,069 INFO [Thread-1976 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1976. 2024-11-18T20:43:22,069 DEBUG [Thread-1974 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:43:22,069 DEBUG [Thread-1974 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:22,069 DEBUG [Thread-1976 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:43:22,069 DEBUG [Thread-1976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:22,069 DEBUG [Thread-1976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:22,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:22,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:22,069 DEBUG [Thread-1974 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:22,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742342_1518 (size=12301) 2024-11-18T20:43:22,076 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/88717b0471e345f0b2019b6111130ecb as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/88717b0471e345f0b2019b6111130ecb 2024-11-18T20:43:22,080 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22cccb73b6312f1ad45031615ae4847f/C of 22cccb73b6312f1ad45031615ae4847f into 88717b0471e345f0b2019b6111130ecb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:22,080 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:22,080 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., storeName=22cccb73b6312f1ad45031615ae4847f/C, priority=13, startTime=1731962601586; duration=0sec 2024-11-18T20:43:22,080 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:22,080 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22cccb73b6312f1ad45031615ae4847f:C 2024-11-18T20:43:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:22,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. as already flushing 2024-11-18T20:43:22,089 INFO [Thread-1971 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1971. 2024-11-18T20:43:22,089 DEBUG [Thread-1971 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:43:22,089 DEBUG [Thread-1971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:22,089 DEBUG [Thread-1971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:22,089 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:43:22,472 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/3cade95168564923bbf5ba385a08cfd2 2024-11-18T20:43:22,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/455baccccba441b983f56ad7f5b3a1c3 is 50, key is test_row_0/C:col10/1731962600859/Put/seqid=0 2024-11-18T20:43:22,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742343_1519 (size=12301) 2024-11-18T20:43:22,887 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/455baccccba441b983f56ad7f5b3a1c3 2024-11-18T20:43:22,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/7e0406ce5ee24aaba212033d85f0a7ba as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/7e0406ce5ee24aaba212033d85f0a7ba 2024-11-18T20:43:22,897 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/7e0406ce5ee24aaba212033d85f0a7ba, entries=150, sequenceid=447, filesize=12.0 K 2024-11-18T20:43:22,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/3cade95168564923bbf5ba385a08cfd2 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/3cade95168564923bbf5ba385a08cfd2 2024-11-18T20:43:22,903 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/3cade95168564923bbf5ba385a08cfd2, entries=150, sequenceid=447, filesize=12.0 K 2024-11-18T20:43:22,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/455baccccba441b983f56ad7f5b3a1c3 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/455baccccba441b983f56ad7f5b3a1c3 2024-11-18T20:43:22,908 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/455baccccba441b983f56ad7f5b3a1c3, entries=150, sequenceid=447, filesize=12.0 K 2024-11-18T20:43:22,909 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=6.71 KB/6870 for 22cccb73b6312f1ad45031615ae4847f in 1308ms, sequenceid=447, compaction requested=false 2024-11-18T20:43:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 22cccb73b6312f1ad45031615ae4847f: 2024-11-18T20:43:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-18T20:43:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-18T20:43:22,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-18T20:43:22,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7670 sec 2024-11-18T20:43:22,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.7740 sec 2024-11-18T20:43:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:43:23,297 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:24,798 INFO [Thread-1969 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1969. 2024-11-18T20:43:24,798 DEBUG [Thread-1969 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:43:24,798 DEBUG [Thread-1969 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:24,799 DEBUG [Thread-1969 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:24,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:24,871 INFO [Thread-1967 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1967. 2024-11-18T20:43:24,871 DEBUG [Thread-1967 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:43:24,871 DEBUG [Thread-1967 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:24,871 DEBUG [Thread-1967 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:24,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:31,049 INFO [Thread-1965 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1965. 2024-11-18T20:43:31,049 DEBUG [Thread-1965 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:43:31,050 DEBUG [Thread-1965 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:31,050 DEBUG [Thread-1965 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:31,050 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:31,101 INFO [Thread-1963 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1963. 2024-11-18T20:43:31,101 DEBUG [Thread-1963 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:43:31,101 DEBUG [Thread-1963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:31,101 DEBUG [Thread-1963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:31,101 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:31,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T20:43:31,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-18T20:43:31,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-18T20:43:31,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-18T20:43:31,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-18T20:43:31,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 173 2024-11-18T20:43:31,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T20:43:31,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T20:43:31,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1178 2024-11-18T20:43:31,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3534 rows 2024-11-18T20:43:31,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1178 2024-11-18T20:43:31,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3533 rows 2024-11-18T20:43:31,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1176 2024-11-18T20:43:31,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3523 rows 2024-11-18T20:43:31,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1180 2024-11-18T20:43:31,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3533 rows 2024-11-18T20:43:31,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1168 2024-11-18T20:43:31,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3503 rows 2024-11-18T20:43:31,102 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:43:31,102 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:43:31,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:31,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:31,103 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:31,104 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T20:43:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T20:43:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T20:43:31,108 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962611108"}]},"ts":"1731962611108"} 2024-11-18T20:43:31,110 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T20:43:31,110 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T20:43:31,111 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:43:31,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22cccb73b6312f1ad45031615ae4847f, UNASSIGN}] 2024-11-18T20:43:31,113 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22cccb73b6312f1ad45031615ae4847f, UNASSIGN 2024-11-18T20:43:31,113 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=22cccb73b6312f1ad45031615ae4847f, regionState=CLOSING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:31,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22cccb73b6312f1ad45031615ae4847f, UNASSIGN because future has completed 2024-11-18T20:43:31,115 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:43:31,115 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:43:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T20:43:31,267 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:31,267 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:43:31,267 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing 22cccb73b6312f1ad45031615ae4847f, disabling compactions & flushes 2024-11-18T20:43:31,267 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:31,267 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:31,267 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. after waiting 0 ms 2024-11-18T20:43:31,267 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:31,267 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing 22cccb73b6312f1ad45031615ae4847f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-18T20:43:31,268 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=A 2024-11-18T20:43:31,268 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:31,268 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=B 2024-11-18T20:43:31,268 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:31,268 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22cccb73b6312f1ad45031615ae4847f, store=C 2024-11-18T20:43:31,268 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:31,272 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/e02a4481d8fd4d24892d30e4ba6b40d1 is 50, key is test_row_0/A:col10/1731962604797/Put/seqid=0 2024-11-18T20:43:31,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742344_1520 (size=12301) 2024-11-18T20:43:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T20:43:31,679 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/e02a4481d8fd4d24892d30e4ba6b40d1 2024-11-18T20:43:31,690 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/2569872b585b445897802aba7771e070 is 50, key is test_row_0/B:col10/1731962604797/Put/seqid=0 2024-11-18T20:43:31,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742345_1521 (size=12301) 2024-11-18T20:43:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T20:43:32,094 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/2569872b585b445897802aba7771e070 2024-11-18T20:43:32,102 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/d061a487cd704576abf1fc668271734e is 50, key is test_row_0/C:col10/1731962604797/Put/seqid=0 2024-11-18T20:43:32,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742346_1522 (size=12301) 2024-11-18T20:43:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T20:43:32,506 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/d061a487cd704576abf1fc668271734e 2024-11-18T20:43:32,512 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/A/e02a4481d8fd4d24892d30e4ba6b40d1 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/e02a4481d8fd4d24892d30e4ba6b40d1 2024-11-18T20:43:32,517 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/e02a4481d8fd4d24892d30e4ba6b40d1, entries=150, sequenceid=458, filesize=12.0 K 2024-11-18T20:43:32,518 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/B/2569872b585b445897802aba7771e070 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/2569872b585b445897802aba7771e070 2024-11-18T20:43:32,522 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/2569872b585b445897802aba7771e070, entries=150, sequenceid=458, filesize=12.0 K 2024-11-18T20:43:32,523 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/.tmp/C/d061a487cd704576abf1fc668271734e as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/d061a487cd704576abf1fc668271734e 2024-11-18T20:43:32,528 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/d061a487cd704576abf1fc668271734e, entries=150, sequenceid=458, filesize=12.0 K 2024-11-18T20:43:32,529 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 22cccb73b6312f1ad45031615ae4847f in 1262ms, sequenceid=458, compaction requested=true 2024-11-18T20:43:32,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/66712570f2b34b2fbeb0dd820329669d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/b1f4e3b47dc84346bec2c4e8bcd6b64c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/6f969d5582234a9893be2626c6c63194, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/a949afb50ef244a3bfca95ee227fd556, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/21d34c613cb6453a9b75f7a193df17c2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/9f108fafdff742af8c67f387fb4b5e48, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/28806fbec4884dfeacb0d3a82eaa4bcb, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/1bdf7aff363a4425900239cfe31a3770, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/4404fdefe1bf47a4ac66f4731ed862b2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/f7b546633ae04eb9bd35a7d7dd9a1e67, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/0ba14a996aa34273940b5d32f1ba7d1b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/52bfa360a39b4ee583e2034015eb88bf, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/cac31bfa3f8c4c3e9f8d6740a3d1f3c6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/fe7cbed39650404eb51cfc7a84aa10ff, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/8e4813976c6441538d1ceeec3002d8b8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/aa1975ef84f24dbbaab34a7de7c5cda7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/b7ad49568fa540f7ae1bd2d55799dd19, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/69a54dc4528b4d00a015c3e9614b5b3a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/328d11db81b5419585bd7b927dc3c421, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/2046611bb5ce437fa5c1a39dd65bfc68, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/23838230f37b4b01bc39410a4a6e3e3b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/acdea4c8325c40c1af6ec38d94de5329, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/f197682361c04d538baf3c04ef34980e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/04d2447e1c624e7683aacb409e19731a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/9e2b903ce67b4a809fb40afa5ed51e30, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/28b86dff88a849968d324fb6d903710d] to archive 2024-11-18T20:43:32,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:43:32,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/66712570f2b34b2fbeb0dd820329669d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/66712570f2b34b2fbeb0dd820329669d 2024-11-18T20:43:32,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/b1f4e3b47dc84346bec2c4e8bcd6b64c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/b1f4e3b47dc84346bec2c4e8bcd6b64c 2024-11-18T20:43:32,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/6f969d5582234a9893be2626c6c63194 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/6f969d5582234a9893be2626c6c63194 2024-11-18T20:43:32,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/a949afb50ef244a3bfca95ee227fd556 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/a949afb50ef244a3bfca95ee227fd556 2024-11-18T20:43:32,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/21d34c613cb6453a9b75f7a193df17c2 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/21d34c613cb6453a9b75f7a193df17c2 2024-11-18T20:43:32,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/9f108fafdff742af8c67f387fb4b5e48 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/9f108fafdff742af8c67f387fb4b5e48 2024-11-18T20:43:32,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/28806fbec4884dfeacb0d3a82eaa4bcb to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/28806fbec4884dfeacb0d3a82eaa4bcb 2024-11-18T20:43:32,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/1bdf7aff363a4425900239cfe31a3770 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/1bdf7aff363a4425900239cfe31a3770 2024-11-18T20:43:32,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/4404fdefe1bf47a4ac66f4731ed862b2 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/4404fdefe1bf47a4ac66f4731ed862b2 2024-11-18T20:43:32,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/f7b546633ae04eb9bd35a7d7dd9a1e67 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/f7b546633ae04eb9bd35a7d7dd9a1e67 2024-11-18T20:43:32,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/0ba14a996aa34273940b5d32f1ba7d1b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/0ba14a996aa34273940b5d32f1ba7d1b 2024-11-18T20:43:32,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/52bfa360a39b4ee583e2034015eb88bf to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/52bfa360a39b4ee583e2034015eb88bf 2024-11-18T20:43:32,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/cac31bfa3f8c4c3e9f8d6740a3d1f3c6 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/cac31bfa3f8c4c3e9f8d6740a3d1f3c6 2024-11-18T20:43:32,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/fe7cbed39650404eb51cfc7a84aa10ff to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/fe7cbed39650404eb51cfc7a84aa10ff 2024-11-18T20:43:32,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/8e4813976c6441538d1ceeec3002d8b8 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/8e4813976c6441538d1ceeec3002d8b8 2024-11-18T20:43:32,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/aa1975ef84f24dbbaab34a7de7c5cda7 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/aa1975ef84f24dbbaab34a7de7c5cda7 2024-11-18T20:43:32,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/b7ad49568fa540f7ae1bd2d55799dd19 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/b7ad49568fa540f7ae1bd2d55799dd19 2024-11-18T20:43:32,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/69a54dc4528b4d00a015c3e9614b5b3a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/69a54dc4528b4d00a015c3e9614b5b3a 2024-11-18T20:43:32,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/328d11db81b5419585bd7b927dc3c421 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/328d11db81b5419585bd7b927dc3c421 2024-11-18T20:43:32,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/2046611bb5ce437fa5c1a39dd65bfc68 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/2046611bb5ce437fa5c1a39dd65bfc68 2024-11-18T20:43:32,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/23838230f37b4b01bc39410a4a6e3e3b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/23838230f37b4b01bc39410a4a6e3e3b 2024-11-18T20:43:32,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/acdea4c8325c40c1af6ec38d94de5329 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/acdea4c8325c40c1af6ec38d94de5329 2024-11-18T20:43:32,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/f197682361c04d538baf3c04ef34980e to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/f197682361c04d538baf3c04ef34980e 2024-11-18T20:43:32,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/04d2447e1c624e7683aacb409e19731a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/04d2447e1c624e7683aacb409e19731a 2024-11-18T20:43:32,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/9e2b903ce67b4a809fb40afa5ed51e30 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/9e2b903ce67b4a809fb40afa5ed51e30 2024-11-18T20:43:32,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/28b86dff88a849968d324fb6d903710d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/28b86dff88a849968d324fb6d903710d 2024-11-18T20:43:32,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/b19375e7e6d44c95a084b9d67cb19fc5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/7fdc81ca77314da89636cd7d69cfcaa3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/f5c0ceb7ef564c7cafb0eecfeae0397c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/60fd0e036c1f484f842ad8eb580bd425, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/53ee90ba07394d17812f811269297268, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/810258417f864e43b4a162595ea8e8b3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/eeb2ec43f6a14bc58381b3b67db0cbb9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/7628568f76e64acc966a856637fcd899, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/fd1a71fcde674caaa55a35b2644d78df, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/40a6f47105734bf5915391a4800321cc, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/86b974fa75a744b28b0546f180df43f9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/5fa75ed40e37473084fbcb5463c4bff3, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/6e21189157d0491da4dc296f7a327413, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/dd5de677956b4e749e166c1f39bbc34f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/e2d3edafc8044131840145a15139cb55, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/fc406c98daa7457e81a4c1c192058795, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/e6a3c22a19574c9294987df6c4b98494, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/2ffde4ae01714b0c8a0fa1c37c6ab607, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/ece7cb5b5375438299b72d18072270e9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/cb98cf04bcee41a181d13122522a27df, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/23199c2c2f0242619dda153bdb04d72c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/1af56fc0034243f2856f9ba7c88eef52, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/94c9929f422e4e52ab7693ff9e31e3ad, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/09ea23f95e8a45ceb0dd6c0d04b5ff70, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/7d2e749cd35a4ffbb75e730a9bd619a4, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/ef9543c73b8145bbbf3fafb2d75067d9] to archive 2024-11-18T20:43:32,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:43:32,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/b19375e7e6d44c95a084b9d67cb19fc5 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/b19375e7e6d44c95a084b9d67cb19fc5 2024-11-18T20:43:32,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/7fdc81ca77314da89636cd7d69cfcaa3 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/7fdc81ca77314da89636cd7d69cfcaa3 2024-11-18T20:43:32,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/f5c0ceb7ef564c7cafb0eecfeae0397c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/f5c0ceb7ef564c7cafb0eecfeae0397c 2024-11-18T20:43:32,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/60fd0e036c1f484f842ad8eb580bd425 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/60fd0e036c1f484f842ad8eb580bd425 2024-11-18T20:43:32,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/53ee90ba07394d17812f811269297268 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/53ee90ba07394d17812f811269297268 2024-11-18T20:43:32,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/810258417f864e43b4a162595ea8e8b3 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/810258417f864e43b4a162595ea8e8b3 2024-11-18T20:43:32,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/eeb2ec43f6a14bc58381b3b67db0cbb9 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/eeb2ec43f6a14bc58381b3b67db0cbb9 2024-11-18T20:43:32,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/7628568f76e64acc966a856637fcd899 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/7628568f76e64acc966a856637fcd899 2024-11-18T20:43:32,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/fd1a71fcde674caaa55a35b2644d78df to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/fd1a71fcde674caaa55a35b2644d78df 2024-11-18T20:43:32,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/40a6f47105734bf5915391a4800321cc to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/40a6f47105734bf5915391a4800321cc 2024-11-18T20:43:32,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/86b974fa75a744b28b0546f180df43f9 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/86b974fa75a744b28b0546f180df43f9 2024-11-18T20:43:32,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/5fa75ed40e37473084fbcb5463c4bff3 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/5fa75ed40e37473084fbcb5463c4bff3 2024-11-18T20:43:32,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/6e21189157d0491da4dc296f7a327413 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/6e21189157d0491da4dc296f7a327413 2024-11-18T20:43:32,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/dd5de677956b4e749e166c1f39bbc34f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/dd5de677956b4e749e166c1f39bbc34f 2024-11-18T20:43:32,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/e2d3edafc8044131840145a15139cb55 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/e2d3edafc8044131840145a15139cb55 2024-11-18T20:43:32,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/fc406c98daa7457e81a4c1c192058795 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/fc406c98daa7457e81a4c1c192058795 2024-11-18T20:43:32,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/e6a3c22a19574c9294987df6c4b98494 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/e6a3c22a19574c9294987df6c4b98494 2024-11-18T20:43:32,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/2ffde4ae01714b0c8a0fa1c37c6ab607 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/2ffde4ae01714b0c8a0fa1c37c6ab607 2024-11-18T20:43:32,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/ece7cb5b5375438299b72d18072270e9 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/ece7cb5b5375438299b72d18072270e9 2024-11-18T20:43:32,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/cb98cf04bcee41a181d13122522a27df to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/cb98cf04bcee41a181d13122522a27df 2024-11-18T20:43:32,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/23199c2c2f0242619dda153bdb04d72c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/23199c2c2f0242619dda153bdb04d72c 2024-11-18T20:43:32,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/1af56fc0034243f2856f9ba7c88eef52 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/1af56fc0034243f2856f9ba7c88eef52 2024-11-18T20:43:32,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/94c9929f422e4e52ab7693ff9e31e3ad to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/94c9929f422e4e52ab7693ff9e31e3ad 2024-11-18T20:43:32,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/09ea23f95e8a45ceb0dd6c0d04b5ff70 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/09ea23f95e8a45ceb0dd6c0d04b5ff70 2024-11-18T20:43:32,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/7d2e749cd35a4ffbb75e730a9bd619a4 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/7d2e749cd35a4ffbb75e730a9bd619a4 2024-11-18T20:43:32,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/ef9543c73b8145bbbf3fafb2d75067d9 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/ef9543c73b8145bbbf3fafb2d75067d9 2024-11-18T20:43:32,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/555a438f55234b02ab10d62c91bb984e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/c0eb2075dfa245aebe186f34d81c212b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/04b92846b9ac4dc394104ec32d8981c2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/399e1fffbe944c06889b26f4cb5fb5a6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/556f05a306ce4aaca60e1a14d408efe5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/21c34857a9fa41d0a0b2a59e5cc7c0e0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/2596a6a0a0994a1da21f10ac6d12f1b2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/9b830bb5e32846789a0f2addfcda1e01, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/9a352d45fa034f62bce432b77b111549, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/528e78ea53a142b7a5e7da9d56e8a4ae, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/bec00e035e01410caa039192cfd9d7d5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/1d284a3efb6f4d91a60a8bc7a03574c7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/b7e8c4b7e22b455e8f3faed8a8f09779, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/f8d03d6a0b8842458a4a43316ba63031, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/5a2a575d556246c2b1c8e0d6b046b6b9, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/a2ce6f495b9b4817989492ca978d688f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/45ea6803b1cd4faf8b2eea997fedc6db, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/5345d82ba035435c8f04371cf717c16a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/65f5f8c057fc4b3a93f5ce42b118d7df, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/b3113399747b485eb4d154c80bb0bb1f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/fd1f6bb9f2bc48d3bf2f0b7e3a1c029f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/7623bd36d6334aa28617199a21dc427a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/1d955d967dac4382a317aac9192d9886, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/2cfd9b0085af4a60a0fc8f3cc75a7664, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/2f19a2b4b3db40aaba6e57f33dbea41a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/c6ecc9217d9e447690282630cda7c8f6] to archive 2024-11-18T20:43:32,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:43:32,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/555a438f55234b02ab10d62c91bb984e to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/555a438f55234b02ab10d62c91bb984e 2024-11-18T20:43:32,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/c0eb2075dfa245aebe186f34d81c212b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/c0eb2075dfa245aebe186f34d81c212b 2024-11-18T20:43:32,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/04b92846b9ac4dc394104ec32d8981c2 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/04b92846b9ac4dc394104ec32d8981c2 2024-11-18T20:43:32,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/399e1fffbe944c06889b26f4cb5fb5a6 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/399e1fffbe944c06889b26f4cb5fb5a6 2024-11-18T20:43:32,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/556f05a306ce4aaca60e1a14d408efe5 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/556f05a306ce4aaca60e1a14d408efe5 2024-11-18T20:43:32,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/21c34857a9fa41d0a0b2a59e5cc7c0e0 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/21c34857a9fa41d0a0b2a59e5cc7c0e0 2024-11-18T20:43:32,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/2596a6a0a0994a1da21f10ac6d12f1b2 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/2596a6a0a0994a1da21f10ac6d12f1b2 2024-11-18T20:43:32,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/9b830bb5e32846789a0f2addfcda1e01 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/9b830bb5e32846789a0f2addfcda1e01 2024-11-18T20:43:32,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/9a352d45fa034f62bce432b77b111549 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/9a352d45fa034f62bce432b77b111549 2024-11-18T20:43:32,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/528e78ea53a142b7a5e7da9d56e8a4ae to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/528e78ea53a142b7a5e7da9d56e8a4ae 2024-11-18T20:43:32,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/bec00e035e01410caa039192cfd9d7d5 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/bec00e035e01410caa039192cfd9d7d5 2024-11-18T20:43:32,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/1d284a3efb6f4d91a60a8bc7a03574c7 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/1d284a3efb6f4d91a60a8bc7a03574c7 2024-11-18T20:43:32,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/b7e8c4b7e22b455e8f3faed8a8f09779 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/b7e8c4b7e22b455e8f3faed8a8f09779 2024-11-18T20:43:32,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/f8d03d6a0b8842458a4a43316ba63031 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/f8d03d6a0b8842458a4a43316ba63031 2024-11-18T20:43:32,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/5a2a575d556246c2b1c8e0d6b046b6b9 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/5a2a575d556246c2b1c8e0d6b046b6b9 2024-11-18T20:43:32,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/a2ce6f495b9b4817989492ca978d688f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/a2ce6f495b9b4817989492ca978d688f 2024-11-18T20:43:32,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/45ea6803b1cd4faf8b2eea997fedc6db to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/45ea6803b1cd4faf8b2eea997fedc6db 2024-11-18T20:43:32,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/5345d82ba035435c8f04371cf717c16a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/5345d82ba035435c8f04371cf717c16a 2024-11-18T20:43:32,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/65f5f8c057fc4b3a93f5ce42b118d7df to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/65f5f8c057fc4b3a93f5ce42b118d7df 2024-11-18T20:43:32,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/b3113399747b485eb4d154c80bb0bb1f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/b3113399747b485eb4d154c80bb0bb1f 2024-11-18T20:43:32,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/fd1f6bb9f2bc48d3bf2f0b7e3a1c029f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/fd1f6bb9f2bc48d3bf2f0b7e3a1c029f 2024-11-18T20:43:32,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/7623bd36d6334aa28617199a21dc427a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/7623bd36d6334aa28617199a21dc427a 2024-11-18T20:43:32,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/1d955d967dac4382a317aac9192d9886 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/1d955d967dac4382a317aac9192d9886 2024-11-18T20:43:32,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/2cfd9b0085af4a60a0fc8f3cc75a7664 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/2cfd9b0085af4a60a0fc8f3cc75a7664 2024-11-18T20:43:32,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/2f19a2b4b3db40aaba6e57f33dbea41a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/2f19a2b4b3db40aaba6e57f33dbea41a 2024-11-18T20:43:32,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/c6ecc9217d9e447690282630cda7c8f6 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/c6ecc9217d9e447690282630cda7c8f6 2024-11-18T20:43:32,632 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/recovered.edits/461.seqid, newMaxSeqId=461, maxSeqId=1 2024-11-18T20:43:32,633 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. 2024-11-18T20:43:32,633 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for 22cccb73b6312f1ad45031615ae4847f: Waiting for close lock at 1731962611267Running coprocessor pre-close hooks at 1731962611267Disabling compacts and flushes for region at 1731962611267Disabling writes for close at 1731962611267Obtaining lock to block concurrent updates at 1731962611267Preparing flush snapshotting stores in 22cccb73b6312f1ad45031615ae4847f at 1731962611267Finished memstore snapshotting TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731962611268 (+1 ms)Flushing stores of TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f. at 1731962611268Flushing 22cccb73b6312f1ad45031615ae4847f/A: creating writer at 1731962611268Flushing 22cccb73b6312f1ad45031615ae4847f/A: appending metadata at 1731962611271 (+3 ms)Flushing 22cccb73b6312f1ad45031615ae4847f/A: closing flushed file at 1731962611272 (+1 ms)Flushing 22cccb73b6312f1ad45031615ae4847f/B: creating writer at 1731962611687 (+415 ms)Flushing 22cccb73b6312f1ad45031615ae4847f/B: appending metadata at 1731962611690 (+3 ms)Flushing 22cccb73b6312f1ad45031615ae4847f/B: closing flushed file at 1731962611690Flushing 22cccb73b6312f1ad45031615ae4847f/C: creating writer at 1731962612098 (+408 ms)Flushing 22cccb73b6312f1ad45031615ae4847f/C: appending metadata at 1731962612101 (+3 ms)Flushing 22cccb73b6312f1ad45031615ae4847f/C: closing flushed file at 1731962612101Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b32c65: reopening flushed file at 1731962612511 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@88f1edd: reopening flushed file at 1731962612517 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@361d6caf: reopening flushed file at 1731962612522 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 22cccb73b6312f1ad45031615ae4847f in 1262ms, sequenceid=458, compaction requested=true at 1731962612529 (+7 ms)Writing region close event to WAL at 1731962612629 (+100 ms)Running coprocessor post-close hooks at 1731962612633 (+4 ms)Closed at 1731962612633 2024-11-18T20:43:32,635 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed 22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:32,636 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=22cccb73b6312f1ad45031615ae4847f, regionState=CLOSED 2024-11-18T20:43:32,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 because future has completed 2024-11-18T20:43:32,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-18T20:43:32,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure 22cccb73b6312f1ad45031615ae4847f, server=c0a89b2656d4,45393,1731962440107 in 1.5230 sec 2024-11-18T20:43:32,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-18T20:43:32,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22cccb73b6312f1ad45031615ae4847f, UNASSIGN in 1.5280 sec 2024-11-18T20:43:32,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-18T20:43:32,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5320 sec 2024-11-18T20:43:32,645 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962612645"}]},"ts":"1731962612645"} 2024-11-18T20:43:32,647 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T20:43:32,647 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T20:43:32,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5440 sec 2024-11-18T20:43:33,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T20:43:33,268 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:33,269 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T20:43:33,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:43:33,272 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:43:33,273 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:43:33,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-18T20:43:33,277 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:33,279 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A, FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B, FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C, FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/recovered.edits] 2024-11-18T20:43:33,282 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/7e0406ce5ee24aaba212033d85f0a7ba to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/7e0406ce5ee24aaba212033d85f0a7ba 2024-11-18T20:43:33,283 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/8a6e2094434e4044994a49fb859dd9f9 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/8a6e2094434e4044994a49fb859dd9f9 2024-11-18T20:43:33,284 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/e02a4481d8fd4d24892d30e4ba6b40d1 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/A/e02a4481d8fd4d24892d30e4ba6b40d1 2024-11-18T20:43:33,287 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/2569872b585b445897802aba7771e070 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/2569872b585b445897802aba7771e070 2024-11-18T20:43:33,288 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/2c5ddfb5cc874fbc86f4e4c08144a90a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/2c5ddfb5cc874fbc86f4e4c08144a90a 2024-11-18T20:43:33,289 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/3cade95168564923bbf5ba385a08cfd2 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/B/3cade95168564923bbf5ba385a08cfd2 2024-11-18T20:43:33,292 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/455baccccba441b983f56ad7f5b3a1c3 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/455baccccba441b983f56ad7f5b3a1c3 2024-11-18T20:43:33,294 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/88717b0471e345f0b2019b6111130ecb to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/88717b0471e345f0b2019b6111130ecb 2024-11-18T20:43:33,295 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/d061a487cd704576abf1fc668271734e to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/C/d061a487cd704576abf1fc668271734e 2024-11-18T20:43:33,299 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/recovered.edits/461.seqid to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f/recovered.edits/461.seqid 2024-11-18T20:43:33,299 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/22cccb73b6312f1ad45031615ae4847f 2024-11-18T20:43:33,299 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T20:43:33,302 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:43:33,304 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T20:43:33,306 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T20:43:33,307 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:43:33,307 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T20:43:33,307 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731962613307"}]},"ts":"9223372036854775807"} 2024-11-18T20:43:33,309 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T20:43:33,309 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 22cccb73b6312f1ad45031615ae4847f, NAME => 'TestAcidGuarantees,,1731962579855.22cccb73b6312f1ad45031615ae4847f.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T20:43:33,309 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T20:43:33,309 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731962613309"}]},"ts":"9223372036854775807"} 2024-11-18T20:43:33,310 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T20:43:33,311 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:43:33,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-11-18T20:43:33,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-18T20:43:33,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T20:43:33,378 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:33,389 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=246 (was 248), OpenFileDescriptor=441 (was 443), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=322 (was 372), ProcessCount=11 (was 11), AvailableMemoryMB=389 (was 1041) 2024-11-18T20:43:33,399 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=322, ProcessCount=11, AvailableMemoryMB=389 2024-11-18T20:43:33,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:43:33,401 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:43:33,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T20:43:33,403 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T20:43:33,403 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:33,403 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-11-18T20:43:33,404 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T20:43:33,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T20:43:33,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742347_1523 (size=1061) 2024-11-18T20:43:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T20:43:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T20:43:33,812 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093 2024-11-18T20:43:33,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742348_1524 (size=53) 2024-11-18T20:43:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T20:43:34,218 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:43:34,218 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing db394ce1aa4c72690b92156fd4dc5d2d, disabling compactions & flushes 2024-11-18T20:43:34,218 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:34,218 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:34,218 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. after waiting 0 ms 2024-11-18T20:43:34,218 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:34,218 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:34,218 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for db394ce1aa4c72690b92156fd4dc5d2d: Waiting for close lock at 1731962614218Disabling compacts and flushes for region at 1731962614218Disabling writes for close at 1731962614218Writing region close event to WAL at 1731962614218Closed at 1731962614218 2024-11-18T20:43:34,219 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T20:43:34,219 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731962614219"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731962614219"}]},"ts":"1731962614219"} 2024-11-18T20:43:34,221 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T20:43:34,222 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T20:43:34,222 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962614222"}]},"ts":"1731962614222"} 2024-11-18T20:43:34,223 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T20:43:34,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=db394ce1aa4c72690b92156fd4dc5d2d, ASSIGN}] 2024-11-18T20:43:34,225 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=db394ce1aa4c72690b92156fd4dc5d2d, ASSIGN 2024-11-18T20:43:34,225 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=db394ce1aa4c72690b92156fd4dc5d2d, ASSIGN; state=OFFLINE, location=c0a89b2656d4,45393,1731962440107; forceNewPlan=false, retain=false 2024-11-18T20:43:34,376 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=db394ce1aa4c72690b92156fd4dc5d2d, regionState=OPENING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:34,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=db394ce1aa4c72690b92156fd4dc5d2d, ASSIGN because future has completed 2024-11-18T20:43:34,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:43:34,533 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:34,534 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:43:34,534 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:34,534 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:43:34,534 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:34,534 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:34,535 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:34,537 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:43:34,537 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db394ce1aa4c72690b92156fd4dc5d2d columnFamilyName A 2024-11-18T20:43:34,537 DEBUG [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:34,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T20:43:34,538 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] regionserver.HStore(327): Store=db394ce1aa4c72690b92156fd4dc5d2d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:43:34,538 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:34,539 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:43:34,539 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db394ce1aa4c72690b92156fd4dc5d2d columnFamilyName B 2024-11-18T20:43:34,539 DEBUG [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:34,539 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] regionserver.HStore(327): Store=db394ce1aa4c72690b92156fd4dc5d2d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:43:34,539 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:34,540 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:43:34,540 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db394ce1aa4c72690b92156fd4dc5d2d columnFamilyName C 2024-11-18T20:43:34,540 DEBUG [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:34,541 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] regionserver.HStore(327): Store=db394ce1aa4c72690b92156fd4dc5d2d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:43:34,541 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:34,541 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:34,542 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:34,542 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:34,543 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:34,543 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:34,544 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:43:34,545 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:34,546 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:43:34,547 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened db394ce1aa4c72690b92156fd4dc5d2d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67986945, jitterRate=0.013084426522254944}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:43:34,547 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:34,547 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for db394ce1aa4c72690b92156fd4dc5d2d: Running coprocessor pre-open hook at 1731962614534Writing region info on filesystem at 1731962614534Initializing all the Stores at 1731962614535 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962614535Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962614535Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962614535Cleaning up temporary data from old regions at 1731962614543 (+8 ms)Running coprocessor post-open hooks at 1731962614547 (+4 ms)Region opened successfully at 1731962614547 2024-11-18T20:43:34,548 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., pid=142, masterSystemTime=1731962614530 2024-11-18T20:43:34,550 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:34,550 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:34,550 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=db394ce1aa4c72690b92156fd4dc5d2d, regionState=OPEN, openSeqNum=2, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:34,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 because future has completed 2024-11-18T20:43:34,555 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-18T20:43:34,555 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 in 174 msec 2024-11-18T20:43:34,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-18T20:43:34,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=db394ce1aa4c72690b92156fd4dc5d2d, ASSIGN in 331 msec 2024-11-18T20:43:34,557 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T20:43:34,557 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962614557"}]},"ts":"1731962614557"} 2024-11-18T20:43:34,558 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T20:43:34,559 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T20:43:34,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1580 sec 2024-11-18T20:43:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T20:43:35,547 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:35,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e17bf09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:35,548 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:35,548 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:35,550 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a9f803e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:35,551 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f1830ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:35,552 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:35,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:35,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:35,554 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:35,555 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:43:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is c0a89b2656d4,46825,1731962439484 2024-11-18T20:43:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6bd7dde9 2024-11-18T20:43:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:43:35,557 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:43:35,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:43:35,559 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:43:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-18T20:43:35,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742349_1525 (size=1097) 2024-11-18T20:43:35,972 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-18T20:43:35,972 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-18T20:43:35,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:43:35,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=db394ce1aa4c72690b92156fd4dc5d2d, REOPEN/MOVE}] 2024-11-18T20:43:35,978 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=db394ce1aa4c72690b92156fd4dc5d2d, REOPEN/MOVE 2024-11-18T20:43:35,979 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=db394ce1aa4c72690b92156fd4dc5d2d, regionState=CLOSING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:35,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=db394ce1aa4c72690b92156fd4dc5d2d, REOPEN/MOVE because future has completed 2024-11-18T20:43:35,981 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:43:35,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:43:36,133 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:36,134 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:43:36,134 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing db394ce1aa4c72690b92156fd4dc5d2d, disabling compactions & flushes 2024-11-18T20:43:36,134 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:36,134 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:36,134 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. after waiting 0 ms 2024-11-18T20:43:36,134 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:36,139 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T20:43:36,140 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:36,140 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for db394ce1aa4c72690b92156fd4dc5d2d: Waiting for close lock at 1731962616134Running coprocessor pre-close hooks at 1731962616134Disabling compacts and flushes for region at 1731962616134Disabling writes for close at 1731962616134Writing region close event to WAL at 1731962616135 (+1 ms)Running coprocessor post-close hooks at 1731962616140 (+5 ms)Closed at 1731962616140 2024-11-18T20:43:36,140 WARN [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: db394ce1aa4c72690b92156fd4dc5d2d to self. 2024-11-18T20:43:36,143 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:36,144 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=db394ce1aa4c72690b92156fd4dc5d2d, regionState=CLOSED 2024-11-18T20:43:36,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 because future has completed 2024-11-18T20:43:36,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-18T20:43:36,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 in 165 msec 2024-11-18T20:43:36,149 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=db394ce1aa4c72690b92156fd4dc5d2d, REOPEN/MOVE; state=CLOSED, location=c0a89b2656d4,45393,1731962440107; forceNewPlan=false, retain=true 2024-11-18T20:43:36,299 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=db394ce1aa4c72690b92156fd4dc5d2d, regionState=OPENING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:36,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=db394ce1aa4c72690b92156fd4dc5d2d, REOPEN/MOVE because future has completed 2024-11-18T20:43:36,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:43:36,456 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:36,457 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:43:36,457 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:36,457 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:43:36,457 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:36,457 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:36,459 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:36,459 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:43:36,459 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db394ce1aa4c72690b92156fd4dc5d2d columnFamilyName A 2024-11-18T20:43:36,460 DEBUG [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:36,461 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] regionserver.HStore(327): Store=db394ce1aa4c72690b92156fd4dc5d2d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:43:36,462 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:36,462 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:43:36,462 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db394ce1aa4c72690b92156fd4dc5d2d columnFamilyName B 2024-11-18T20:43:36,462 DEBUG [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:36,463 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] regionserver.HStore(327): Store=db394ce1aa4c72690b92156fd4dc5d2d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:43:36,463 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:36,463 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T20:43:36,463 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db394ce1aa4c72690b92156fd4dc5d2d columnFamilyName C 2024-11-18T20:43:36,463 DEBUG [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:36,464 INFO [StoreOpener-db394ce1aa4c72690b92156fd4dc5d2d-1 {}] regionserver.HStore(327): Store=db394ce1aa4c72690b92156fd4dc5d2d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:43:36,464 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:36,464 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:36,465 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:36,466 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:36,467 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:36,467 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:36,467 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:43:36,469 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:36,469 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened db394ce1aa4c72690b92156fd4dc5d2d; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60311451, jitterRate=-0.10128934681415558}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:43:36,469 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:36,470 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for db394ce1aa4c72690b92156fd4dc5d2d: Running coprocessor pre-open hook at 1731962616457Writing region info on filesystem at 1731962616457Initializing all the Stores at 1731962616458 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962616458Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962616458Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731962616458Cleaning up temporary data from old regions at 1731962616467 (+9 ms)Running coprocessor post-open hooks at 1731962616469 (+2 ms)Region opened successfully at 1731962616470 (+1 ms) 2024-11-18T20:43:36,471 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., pid=147, masterSystemTime=1731962616454 2024-11-18T20:43:36,473 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:36,473 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:36,474 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=db394ce1aa4c72690b92156fd4dc5d2d, regionState=OPEN, openSeqNum=5, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:36,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 because future has completed 2024-11-18T20:43:36,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-11-18T20:43:36,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 in 175 msec 2024-11-18T20:43:36,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-18T20:43:36,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=db394ce1aa4c72690b92156fd4dc5d2d, REOPEN/MOVE in 502 msec 2024-11-18T20:43:36,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-18T20:43:36,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 507 msec 2024-11-18T20:43:36,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 925 msec 2024-11-18T20:43:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-18T20:43:36,486 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:36,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac41996, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,487 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:36,487 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:36,488 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f2be891, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:36,490 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f9b31c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22aab9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,491 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:36,491 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:36,491 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78762825, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:36,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:36,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:36,494 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:36,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a6a349d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a34ec22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:36,495 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:36,496 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ff555b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:36,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:36,497 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:36,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c7f03ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@380ce272, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,499 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:36,499 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:36,500 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dfa154c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:36,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:36,502 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28e566b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14c11d8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,503 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:36,503 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:36,504 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:36,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:36,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:36,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1622ffd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:36,505 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@159d963, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cc99027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:36,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:36,507 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c27d5c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:36,508 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79653c40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24edf963, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,509 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:36,509 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:36,510 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4745b9a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:36,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:36,511 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36773f89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2966c360, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,512 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:36,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:36,513 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:36,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a64d78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:36,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:36,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:36,515 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:36,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5441bd62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6379f6de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,516 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:36,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:36,516 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:36,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@131c7607, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:36,518 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:36,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2200da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,519 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337ab365, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,519 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,46825,-1 for getting cluster id 2024-11-18T20:43:36,519 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:43:36,520 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9c0a363-374e-42c9-896f-88dfa8b27e5e' 2024-11-18T20:43:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:43:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9c0a363-374e-42c9-896f-88dfa8b27e5e" 2024-11-18T20:43:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54b16b24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,46825,-1] 2024-11-18T20:43:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:43:36,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:36,521 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:43:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c08a187, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:43:36,523 DEBUG [Thread-2367 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:36,523 DEBUG [Thread-2356 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:36,523 DEBUG [Thread-2371 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:36,523 DEBUG [Thread-2362 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:36,523 DEBUG [Thread-2375 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:36,523 DEBUG [Thread-2360 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:36,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:43:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:36,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:36,525 DEBUG [Thread-2373 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:36,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:36,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:36,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:36,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:36,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:36,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:36,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-18T20:43:36,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:36,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:36,527 DEBUG [Thread-2364 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:36,527 DEBUG [Thread-2369 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:36,527 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:43:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:43:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:43:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T20:43:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:43:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:43:36,529 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:43:36,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:36,529 DEBUG [Thread-2358 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:43:36,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:43:36,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:43:36,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:43:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,45393,1731962440107, seqNum=-1] 2024-11-18T20:43:36,531 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:43:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:43:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:43:36,532 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:43:36,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:43:36,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5] 2024-11-18T20:43:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:36,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T20:43:36,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:43:36,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:36,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:43:36,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:36,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:43:36,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:36,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962676558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:36,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962676560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:36,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962676561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:36,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35952 deadline: 1731962676563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:36,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180d310c786d9741fca444c8064d346b9a_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_0/A:col10/1731962616536/Put/seqid=0 2024-11-18T20:43:36,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962676565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:36,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742350_1526 (size=17034) 2024-11-18T20:43:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T20:43:36,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962676667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:36,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:36,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962676677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:36,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962676678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:36,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35952 deadline: 1731962676678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:36,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962676678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:36,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T20:43:36,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:36,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:36,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:36,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:36,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:36,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T20:43:36,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:36,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:36,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:36,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:36,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T20:43:36,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962676877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:36,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:36,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:36,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962676888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:36,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:36,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962676888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:36,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:36,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:36,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35952 deadline: 1731962676888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:36,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:36,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:36,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962676888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:36,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:36,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:36,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:36,983 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:36,986 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180d310c786d9741fca444c8064d346b9a_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180d310c786d9741fca444c8064d346b9a_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:36,987 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/49bdc754670b457cb0191f6e33ff71c8, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:36,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/49bdc754670b457cb0191f6e33ff71c8 is 175, key is test_row_0/A:col10/1731962616536/Put/seqid=0 2024-11-18T20:43:36,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742351_1527 (size=48139) 2024-11-18T20:43:36,991 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/49bdc754670b457cb0191f6e33ff71c8 2024-11-18T20:43:36,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T20:43:36,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:36,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:36,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:36,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:37,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/c38efdabed35477bb5da875d1cedc997 is 50, key is test_row_0/B:col10/1731962616536/Put/seqid=0 2024-11-18T20:43:37,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742352_1528 (size=12001) 2024-11-18T20:43:37,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T20:43:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:37,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T20:43:37,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962677188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:37,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:37,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:37,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962677197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962677197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:37,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:37,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:37,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35952 deadline: 1731962677197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962677198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:37,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T20:43:37,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:37,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:37,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:37,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:37,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:37,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/c38efdabed35477bb5da875d1cedc997 2024-11-18T20:43:37,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/f6918feadb994f8fad680f81226f8b0f is 50, key is test_row_0/C:col10/1731962616536/Put/seqid=0 2024-11-18T20:43:37,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742353_1529 (size=12001) 2024-11-18T20:43:37,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T20:43:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:37,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:37,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T20:43:37,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:37,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:37,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:37,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:37,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:37,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:37,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T20:43:37,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962677697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:37,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962677708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:37,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962677708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35952 deadline: 1731962677708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:37,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962677708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:37,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T20:43:37,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:37,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:37,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:37,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:37,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:37,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/f6918feadb994f8fad680f81226f8b0f 2024-11-18T20:43:37,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/49bdc754670b457cb0191f6e33ff71c8 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/49bdc754670b457cb0191f6e33ff71c8 2024-11-18T20:43:37,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/49bdc754670b457cb0191f6e33ff71c8, entries=250, sequenceid=17, filesize=47.0 K 2024-11-18T20:43:37,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/c38efdabed35477bb5da875d1cedc997 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/c38efdabed35477bb5da875d1cedc997 2024-11-18T20:43:37,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/c38efdabed35477bb5da875d1cedc997, entries=150, sequenceid=17, filesize=11.7 K 2024-11-18T20:43:37,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/f6918feadb994f8fad680f81226f8b0f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/f6918feadb994f8fad680f81226f8b0f 2024-11-18T20:43:37,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/f6918feadb994f8fad680f81226f8b0f, entries=150, sequenceid=17, filesize=11.7 K 2024-11-18T20:43:37,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for db394ce1aa4c72690b92156fd4dc5d2d in 1325ms, sequenceid=17, compaction requested=false 2024-11-18T20:43:37,864 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T20:43:37,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:37,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T20:43:37,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:37,914 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T20:43:37,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:43:37,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:37,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:43:37,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:37,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:43:37,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:37,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b0d7a574c7e24d5e93b403aaaaec2d96_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_0/A:col10/1731962616550/Put/seqid=0 2024-11-18T20:43:37,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742354_1530 (size=12154) 2024-11-18T20:43:38,152 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T20:43:38,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:38,328 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b0d7a574c7e24d5e93b403aaaaec2d96_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b0d7a574c7e24d5e93b403aaaaec2d96_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:38,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/bb7ad99e490e47c6acefab19ed3d5c52, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:38,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/bb7ad99e490e47c6acefab19ed3d5c52 is 175, key is test_row_0/A:col10/1731962616550/Put/seqid=0 2024-11-18T20:43:38,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742355_1531 (size=30955) 2024-11-18T20:43:38,409 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:43:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T20:43:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:38,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:38,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962678724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:38,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962678725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:38,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:38,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:38,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:38,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:38,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:38,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:38,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962678726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:38,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35952 deadline: 1731962678727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:38,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962678727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:38,731 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in db394ce1aa4c72690b92156fd4dc5d2d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:38,736 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/bb7ad99e490e47c6acefab19ed3d5c52 2024-11-18T20:43:38,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/8d5c88e43da94060bd5a147e497ea839 is 50, key is test_row_0/B:col10/1731962616550/Put/seqid=0 2024-11-18T20:43:38,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742356_1532 (size=12001) 2024-11-18T20:43:38,746 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/8d5c88e43da94060bd5a147e497ea839 2024-11-18T20:43:38,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/bb8636b6feea4b79898f17572988f2a1 is 50, key is test_row_0/C:col10/1731962616550/Put/seqid=0 2024-11-18T20:43:38,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742357_1533 (size=12001) 2024-11-18T20:43:38,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:38,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962678837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962678837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:38,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:38,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962678837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962678837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:38,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:38,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:38,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:39,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:39,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962679047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962679047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:39,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:39,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962679048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962679048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:39,155 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/bb8636b6feea4b79898f17572988f2a1 2024-11-18T20:43:39,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/bb7ad99e490e47c6acefab19ed3d5c52 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/bb7ad99e490e47c6acefab19ed3d5c52 2024-11-18T20:43:39,163 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/bb7ad99e490e47c6acefab19ed3d5c52, entries=150, sequenceid=36, filesize=30.2 K 2024-11-18T20:43:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/8d5c88e43da94060bd5a147e497ea839 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/8d5c88e43da94060bd5a147e497ea839 2024-11-18T20:43:39,184 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/8d5c88e43da94060bd5a147e497ea839, entries=150, sequenceid=36, filesize=11.7 K 2024-11-18T20:43:39,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/bb8636b6feea4b79898f17572988f2a1 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/bb8636b6feea4b79898f17572988f2a1 2024-11-18T20:43:39,188 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/bb8636b6feea4b79898f17572988f2a1, entries=150, sequenceid=36, filesize=11.7 K 2024-11-18T20:43:39,189 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for db394ce1aa4c72690b92156fd4dc5d2d in 1274ms, sequenceid=36, compaction requested=false 2024-11-18T20:43:39,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:39,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:39,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-18T20:43:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-18T20:43:39,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-18T20:43:39,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6610 sec 2024-11-18T20:43:39,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.6680 sec 2024-11-18T20:43:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:39,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T20:43:39,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:43:39,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:39,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:43:39,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:39,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:43:39,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:39,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186cdcc8e5d7a74d48bddc6965d10d1686_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_0/A:col10/1731962619359/Put/seqid=0 2024-11-18T20:43:39,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742358_1534 (size=12154) 2024-11-18T20:43:39,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:39,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962679379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962679380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:39,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:39,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:39,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962679383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:39,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962679384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:39,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962679487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962679487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:39,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962679497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:39,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962679497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:39,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962679697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:39,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962679697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:39,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:39,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962679707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962679707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:39,774 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:39,778 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186cdcc8e5d7a74d48bddc6965d10d1686_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186cdcc8e5d7a74d48bddc6965d10d1686_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:39,779 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/c89be9009c8b4503bb5208f138f80db8, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:39,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/c89be9009c8b4503bb5208f138f80db8 is 175, key is test_row_0/A:col10/1731962619359/Put/seqid=0 2024-11-18T20:43:39,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742359_1535 (size=30955) 2024-11-18T20:43:40,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962680007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:40,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:40,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:40,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962680007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:40,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:40,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:40,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:40,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:40,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962680017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:40,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962680018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:40,183 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/c89be9009c8b4503bb5208f138f80db8 2024-11-18T20:43:40,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/870af5227e58480a8de1a7461a428e8b is 50, key is test_row_0/B:col10/1731962619359/Put/seqid=0 2024-11-18T20:43:40,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742360_1536 (size=12001) 2024-11-18T20:43:40,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962680517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:40,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962680517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:40,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962680527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:40,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:40,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962680527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:40,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:40,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:40,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:40,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:40,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:40,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:40,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/870af5227e58480a8de1a7461a428e8b 2024-11-18T20:43:40,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/26fe6c39e88f48a6b5b6f52e117dd5db is 50, key is test_row_0/C:col10/1731962619359/Put/seqid=0 2024-11-18T20:43:40,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742361_1537 (size=12001) 2024-11-18T20:43:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T20:43:40,698 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:40,699 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:43:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-18T20:43:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T20:43:40,700 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:43:40,701 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:43:40,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:43:40,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35952 deadline: 1731962680747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:40,750 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in db394ce1aa4c72690b92156fd4dc5d2d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T20:43:40,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T20:43:40,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:40,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:40,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:40,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:40,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:41,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T20:43:41,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:41,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:41,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:41,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:41,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T20:43:41,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/26fe6c39e88f48a6b5b6f52e117dd5db 2024-11-18T20:43:41,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/c89be9009c8b4503bb5208f138f80db8 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c89be9009c8b4503bb5208f138f80db8 2024-11-18T20:43:41,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c89be9009c8b4503bb5208f138f80db8, entries=150, sequenceid=51, filesize=30.2 K 2024-11-18T20:43:41,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/870af5227e58480a8de1a7461a428e8b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/870af5227e58480a8de1a7461a428e8b 2024-11-18T20:43:41,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/870af5227e58480a8de1a7461a428e8b, entries=150, sequenceid=51, filesize=11.7 K 2024-11-18T20:43:41,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/26fe6c39e88f48a6b5b6f52e117dd5db as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/26fe6c39e88f48a6b5b6f52e117dd5db 2024-11-18T20:43:41,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/26fe6c39e88f48a6b5b6f52e117dd5db, entries=150, sequenceid=51, filesize=11.7 K 2024-11-18T20:43:41,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for db394ce1aa4c72690b92156fd4dc5d2d in 1690ms, sequenceid=51, compaction requested=true 2024-11-18T20:43:41,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:41,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:43:41,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:41,051 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:41,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:43:41,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:41,051 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:41,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:43:41,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:41,052 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:41,052 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:41,052 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/A is initiating minor compaction (all files) 2024-11-18T20:43:41,052 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/B is initiating minor compaction (all files) 2024-11-18T20:43:41,053 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/B in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:41,053 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/A in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:41,053 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/49bdc754670b457cb0191f6e33ff71c8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/bb7ad99e490e47c6acefab19ed3d5c52, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c89be9009c8b4503bb5208f138f80db8] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=107.5 K 2024-11-18T20:43:41,053 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/c38efdabed35477bb5da875d1cedc997, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/8d5c88e43da94060bd5a147e497ea839, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/870af5227e58480a8de1a7461a428e8b] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=35.2 K 2024-11-18T20:43:41,053 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:41,053 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/49bdc754670b457cb0191f6e33ff71c8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/bb7ad99e490e47c6acefab19ed3d5c52, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c89be9009c8b4503bb5208f138f80db8] 2024-11-18T20:43:41,053 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting c38efdabed35477bb5da875d1cedc997, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731962616534 2024-11-18T20:43:41,053 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49bdc754670b457cb0191f6e33ff71c8, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731962616534 2024-11-18T20:43:41,053 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d5c88e43da94060bd5a147e497ea839, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731962616550 2024-11-18T20:43:41,053 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb7ad99e490e47c6acefab19ed3d5c52, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731962616550 2024-11-18T20:43:41,053 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 870af5227e58480a8de1a7461a428e8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731962618722 2024-11-18T20:43:41,053 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting c89be9009c8b4503bb5208f138f80db8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731962618722 2024-11-18T20:43:41,066 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:41,068 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118e826fe68c255402caf22650a88bd6b1c_db394ce1aa4c72690b92156fd4dc5d2d store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:41,069 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#B#compaction#461 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:41,070 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/b367ea65bb8844afb7f5ffd5f5444a60 is 50, key is test_row_0/B:col10/1731962619359/Put/seqid=0 2024-11-18T20:43:41,071 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118e826fe68c255402caf22650a88bd6b1c_db394ce1aa4c72690b92156fd4dc5d2d, store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:41,071 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e826fe68c255402caf22650a88bd6b1c_db394ce1aa4c72690b92156fd4dc5d2d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:41,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742362_1538 (size=12104) 2024-11-18T20:43:41,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742363_1539 (size=4469) 2024-11-18T20:43:41,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T20:43:41,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:41,163 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T20:43:41,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:43:41,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:41,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:43:41,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:41,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:43:41,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:41,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ed5edd25b93c417fb9c7d9c431605f79_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_0/A:col10/1731962619374/Put/seqid=0 2024-11-18T20:43:41,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742364_1540 (size=12154) 2024-11-18T20:43:41,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:41,181 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ed5edd25b93c417fb9c7d9c431605f79_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ed5edd25b93c417fb9c7d9c431605f79_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:41,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/d1351b88e6a94c38b7d1c37689025037, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:41,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/d1351b88e6a94c38b7d1c37689025037 is 175, key is test_row_0/A:col10/1731962619374/Put/seqid=0 2024-11-18T20:43:41,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742365_1541 (size=30955) 2024-11-18T20:43:41,189 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/d1351b88e6a94c38b7d1c37689025037 2024-11-18T20:43:41,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/32b2ac82079245eda3b0bf65e51c98d5 is 50, key is test_row_0/B:col10/1731962619374/Put/seqid=0 2024-11-18T20:43:41,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742366_1542 (size=12001) 2024-11-18T20:43:41,203 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/32b2ac82079245eda3b0bf65e51c98d5 2024-11-18T20:43:41,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/ea464c81a53349c9a0670ed9649a6952 is 50, key is test_row_0/C:col10/1731962619374/Put/seqid=0 2024-11-18T20:43:41,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742367_1543 (size=12001) 2024-11-18T20:43:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T20:43:41,490 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/b367ea65bb8844afb7f5ffd5f5444a60 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/b367ea65bb8844afb7f5ffd5f5444a60 2024-11-18T20:43:41,494 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/B of db394ce1aa4c72690b92156fd4dc5d2d into b367ea65bb8844afb7f5ffd5f5444a60(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:41,494 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:41,494 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/B, priority=13, startTime=1731962621051; duration=0sec 2024-11-18T20:43:41,494 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:41,494 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:B 2024-11-18T20:43:41,494 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:41,495 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:41,495 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/C is initiating minor compaction (all files) 2024-11-18T20:43:41,495 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/C in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:41,495 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/f6918feadb994f8fad680f81226f8b0f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/bb8636b6feea4b79898f17572988f2a1, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/26fe6c39e88f48a6b5b6f52e117dd5db] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=35.2 K 2024-11-18T20:43:41,495 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting f6918feadb994f8fad680f81226f8b0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731962616534 2024-11-18T20:43:41,495 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting bb8636b6feea4b79898f17572988f2a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731962616550 2024-11-18T20:43:41,496 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 26fe6c39e88f48a6b5b6f52e117dd5db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731962618722 2024-11-18T20:43:41,499 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#A#compaction#462 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:41,499 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/c4690563cd7347bba28bb0b0ac36658f is 175, key is test_row_0/A:col10/1731962619359/Put/seqid=0 2024-11-18T20:43:41,501 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#C#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:41,501 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/931854dead8044948bfc250472dbe16a is 50, key is test_row_0/C:col10/1731962619359/Put/seqid=0 2024-11-18T20:43:41,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742368_1544 (size=31058) 2024-11-18T20:43:41,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742369_1545 (size=12104) 2024-11-18T20:43:41,508 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/931854dead8044948bfc250472dbe16a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/931854dead8044948bfc250472dbe16a 2024-11-18T20:43:41,512 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/C of db394ce1aa4c72690b92156fd4dc5d2d into 931854dead8044948bfc250472dbe16a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:41,512 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:41,512 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/C, priority=13, startTime=1731962621051; duration=0sec 2024-11-18T20:43:41,512 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:41,512 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:C 2024-11-18T20:43:41,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:41,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962681547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:41,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962681548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:41,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:41,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962681548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:41,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962681549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:41,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:41,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:41,613 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/ea464c81a53349c9a0670ed9649a6952 2024-11-18T20:43:41,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/d1351b88e6a94c38b7d1c37689025037 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/d1351b88e6a94c38b7d1c37689025037 2024-11-18T20:43:41,621 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/d1351b88e6a94c38b7d1c37689025037, entries=150, sequenceid=68, filesize=30.2 K 2024-11-18T20:43:41,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/32b2ac82079245eda3b0bf65e51c98d5 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/32b2ac82079245eda3b0bf65e51c98d5 2024-11-18T20:43:41,625 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/32b2ac82079245eda3b0bf65e51c98d5, entries=150, sequenceid=68, filesize=11.7 K 2024-11-18T20:43:41,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/ea464c81a53349c9a0670ed9649a6952 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/ea464c81a53349c9a0670ed9649a6952 2024-11-18T20:43:41,629 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/ea464c81a53349c9a0670ed9649a6952, entries=150, sequenceid=68, filesize=11.7 K 2024-11-18T20:43:41,629 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for db394ce1aa4c72690b92156fd4dc5d2d in 466ms, sequenceid=68, compaction requested=false 2024-11-18T20:43:41,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:41,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:41,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-18T20:43:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-18T20:43:41,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-18T20:43:41,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 930 msec 2024-11-18T20:43:41,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 935 msec 2024-11-18T20:43:41,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:41,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T20:43:41,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:43:41,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:41,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:43:41,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:41,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:43:41,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:41,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111807cdfee7e46c49baa7a232d1dcd19f4e_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_0/A:col10/1731962621660/Put/seqid=0 2024-11-18T20:43:41,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742370_1546 (size=12154) 2024-11-18T20:43:41,674 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:41,677 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111807cdfee7e46c49baa7a232d1dcd19f4e_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111807cdfee7e46c49baa7a232d1dcd19f4e_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:41,678 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/422facf368164c01a20fc89623e89d91, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:41,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/422facf368164c01a20fc89623e89d91 is 175, key is test_row_0/A:col10/1731962621660/Put/seqid=0 2024-11-18T20:43:41,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742371_1547 (size=30955) 2024-11-18T20:43:41,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962681676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:41,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962681683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:41,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962681683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962681683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:41,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962681787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:41,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:41,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962681798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962681797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:41,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962681798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T20:43:41,837 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:41,838 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:43:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-18T20:43:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T20:43:41,839 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:43:41,840 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:43:41,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:43:41,908 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/c4690563cd7347bba28bb0b0ac36658f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c4690563cd7347bba28bb0b0ac36658f 2024-11-18T20:43:41,914 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/A of db394ce1aa4c72690b92156fd4dc5d2d into c4690563cd7347bba28bb0b0ac36658f(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:41,914 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:41,914 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/A, priority=13, startTime=1731962621050; duration=0sec 2024-11-18T20:43:41,914 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:41,914 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:A 2024-11-18T20:43:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T20:43:41,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T20:43:41,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:41,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:41,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:41,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:41,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:42,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962681997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962682007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962682007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962682007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,081 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=87, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/422facf368164c01a20fc89623e89d91 2024-11-18T20:43:42,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/e1ea1ad8cacb4727b09c306ce1727019 is 50, key is test_row_0/B:col10/1731962621660/Put/seqid=0 2024-11-18T20:43:42,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742372_1548 (size=12001) 2024-11-18T20:43:42,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/e1ea1ad8cacb4727b09c306ce1727019 2024-11-18T20:43:42,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/62580eaea1c2400287cc849709c0128a is 50, key is test_row_0/C:col10/1731962621660/Put/seqid=0 2024-11-18T20:43:42,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742373_1549 (size=12001) 2024-11-18T20:43:42,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/62580eaea1c2400287cc849709c0128a 2024-11-18T20:43:42,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/422facf368164c01a20fc89623e89d91 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/422facf368164c01a20fc89623e89d91 2024-11-18T20:43:42,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/422facf368164c01a20fc89623e89d91, entries=150, sequenceid=87, filesize=30.2 K 2024-11-18T20:43:42,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/e1ea1ad8cacb4727b09c306ce1727019 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/e1ea1ad8cacb4727b09c306ce1727019 2024-11-18T20:43:42,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/e1ea1ad8cacb4727b09c306ce1727019, entries=150, sequenceid=87, filesize=11.7 K 2024-11-18T20:43:42,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/62580eaea1c2400287cc849709c0128a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/62580eaea1c2400287cc849709c0128a 2024-11-18T20:43:42,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/62580eaea1c2400287cc849709c0128a, entries=150, sequenceid=87, filesize=11.7 K 2024-11-18T20:43:42,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for db394ce1aa4c72690b92156fd4dc5d2d in 454ms, sequenceid=87, compaction requested=true 2024-11-18T20:43:42,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:42,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:43:42,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:42,116 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:42,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:43:42,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:42,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:43:42,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:42,116 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:42,117 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:42,117 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/B is initiating minor compaction (all files) 2024-11-18T20:43:42,117 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/B in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:42,117 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/b367ea65bb8844afb7f5ffd5f5444a60, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/32b2ac82079245eda3b0bf65e51c98d5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/e1ea1ad8cacb4727b09c306ce1727019] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=35.3 K 2024-11-18T20:43:42,117 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:42,117 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/A is initiating minor compaction (all files) 2024-11-18T20:43:42,117 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/A in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:42,117 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c4690563cd7347bba28bb0b0ac36658f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/d1351b88e6a94c38b7d1c37689025037, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/422facf368164c01a20fc89623e89d91] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=90.8 K 2024-11-18T20:43:42,117 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting b367ea65bb8844afb7f5ffd5f5444a60, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731962618722 2024-11-18T20:43:42,117 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:42,117 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c4690563cd7347bba28bb0b0ac36658f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/d1351b88e6a94c38b7d1c37689025037, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/422facf368164c01a20fc89623e89d91] 2024-11-18T20:43:42,117 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 32b2ac82079245eda3b0bf65e51c98d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731962619374 2024-11-18T20:43:42,117 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4690563cd7347bba28bb0b0ac36658f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731962618722 2024-11-18T20:43:42,118 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting e1ea1ad8cacb4727b09c306ce1727019, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731962621546 2024-11-18T20:43:42,118 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1351b88e6a94c38b7d1c37689025037, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731962619374 2024-11-18T20:43:42,118 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 422facf368164c01a20fc89623e89d91, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731962621546 2024-11-18T20:43:42,123 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:42,124 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#B#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:42,125 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/3e8ec9ec41564d8eae67781bc506a96b is 50, key is test_row_0/B:col10/1731962621660/Put/seqid=0 2024-11-18T20:43:42,126 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411180473115d4afc4bcaa9041819febc6786_db394ce1aa4c72690b92156fd4dc5d2d store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:42,128 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411180473115d4afc4bcaa9041819febc6786_db394ce1aa4c72690b92156fd4dc5d2d, store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:42,128 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180473115d4afc4bcaa9041819febc6786_db394ce1aa4c72690b92156fd4dc5d2d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:42,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742374_1550 (size=12207) 2024-11-18T20:43:42,136 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/3e8ec9ec41564d8eae67781bc506a96b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/3e8ec9ec41564d8eae67781bc506a96b 2024-11-18T20:43:42,140 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/B of db394ce1aa4c72690b92156fd4dc5d2d into 3e8ec9ec41564d8eae67781bc506a96b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:42,140 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:42,140 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/B, priority=13, startTime=1731962622116; duration=0sec 2024-11-18T20:43:42,140 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:42,141 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:B 2024-11-18T20:43:42,141 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:42,141 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:42,141 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/C is initiating minor compaction (all files) 2024-11-18T20:43:42,141 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/C in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:42,142 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/931854dead8044948bfc250472dbe16a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/ea464c81a53349c9a0670ed9649a6952, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/62580eaea1c2400287cc849709c0128a] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=35.3 K 2024-11-18T20:43:42,142 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 931854dead8044948bfc250472dbe16a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731962618722 2024-11-18T20:43:42,143 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting ea464c81a53349c9a0670ed9649a6952, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731962619374 2024-11-18T20:43:42,143 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 62580eaea1c2400287cc849709c0128a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731962621546 2024-11-18T20:43:42,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T20:43:42,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:42,145 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T20:43:42,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:43:42,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:42,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:43:42,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:42,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:43:42,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:42,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742375_1551 (size=4469) 2024-11-18T20:43:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T20:43:42,164 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#C#compaction#472 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:42,164 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/c2b4225f3d1f4362b607b78c4bb2c110 is 50, key is test_row_0/C:col10/1731962621660/Put/seqid=0 2024-11-18T20:43:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188f01a03b3dd54944aedb72c46ba9a840_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_0/A:col10/1731962621681/Put/seqid=0 2024-11-18T20:43:42,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742376_1552 (size=12207) 2024-11-18T20:43:42,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742377_1553 (size=12154) 2024-11-18T20:43:42,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,178 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188f01a03b3dd54944aedb72c46ba9a840_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188f01a03b3dd54944aedb72c46ba9a840_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:42,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/439df09e7bfc4f068ac1a0e09e88e437, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:42,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/439df09e7bfc4f068ac1a0e09e88e437 is 175, key is test_row_0/A:col10/1731962621681/Put/seqid=0 2024-11-18T20:43:42,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742378_1554 (size=30955) 2024-11-18T20:43:42,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:42,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:42,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962682328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962682328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962682332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962682332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962682437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962682438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962682447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962682447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T20:43:42,553 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#A#compaction#471 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:42,553 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/3d85e2fc001d49da9af2f5020557b597 is 175, key is test_row_0/A:col10/1731962621660/Put/seqid=0 2024-11-18T20:43:42,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742379_1555 (size=31161) 2024-11-18T20:43:42,567 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/3d85e2fc001d49da9af2f5020557b597 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/3d85e2fc001d49da9af2f5020557b597 2024-11-18T20:43:42,572 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/A of db394ce1aa4c72690b92156fd4dc5d2d into 3d85e2fc001d49da9af2f5020557b597(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:42,572 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:42,572 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/A, priority=13, startTime=1731962622116; duration=0sec 2024-11-18T20:43:42,572 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:42,572 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:A 2024-11-18T20:43:42,577 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/c2b4225f3d1f4362b607b78c4bb2c110 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c2b4225f3d1f4362b607b78c4bb2c110 2024-11-18T20:43:42,583 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/C of db394ce1aa4c72690b92156fd4dc5d2d into c2b4225f3d1f4362b607b78c4bb2c110(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:42,583 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:42,583 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/C, priority=13, startTime=1731962622116; duration=0sec 2024-11-18T20:43:42,583 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:42,583 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:C 2024-11-18T20:43:42,584 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/439df09e7bfc4f068ac1a0e09e88e437 2024-11-18T20:43:42,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/c15212ece9744c56aa60022363c7c0e5 is 50, key is test_row_0/B:col10/1731962621681/Put/seqid=0 2024-11-18T20:43:42,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742380_1556 (size=12001) 2024-11-18T20:43:42,596 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/c15212ece9744c56aa60022363c7c0e5 2024-11-18T20:43:42,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/744c80bd0a3d4e199401f442e1a032b8 is 50, key is test_row_0/C:col10/1731962621681/Put/seqid=0 2024-11-18T20:43:42,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742381_1557 (size=12001) 2024-11-18T20:43:42,608 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/744c80bd0a3d4e199401f442e1a032b8 2024-11-18T20:43:42,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/439df09e7bfc4f068ac1a0e09e88e437 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/439df09e7bfc4f068ac1a0e09e88e437 2024-11-18T20:43:42,616 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/439df09e7bfc4f068ac1a0e09e88e437, entries=150, sequenceid=105, filesize=30.2 K 2024-11-18T20:43:42,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/c15212ece9744c56aa60022363c7c0e5 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/c15212ece9744c56aa60022363c7c0e5 2024-11-18T20:43:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,620 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/c15212ece9744c56aa60022363c7c0e5, entries=150, sequenceid=105, filesize=11.7 K 2024-11-18T20:43:42,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/744c80bd0a3d4e199401f442e1a032b8 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/744c80bd0a3d4e199401f442e1a032b8 2024-11-18T20:43:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,624 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/744c80bd0a3d4e199401f442e1a032b8, entries=150, sequenceid=105, filesize=11.7 K 2024-11-18T20:43:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,625 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for db394ce1aa4c72690b92156fd4dc5d2d in 480ms, sequenceid=105, compaction requested=false 2024-11-18T20:43:42,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:42,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:42,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-18T20:43:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-18T20:43:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-18T20:43:42,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 786 msec 2024-11-18T20:43:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 791 msec 2024-11-18T20:43:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T20:43:42,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:43:42,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:42,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:43:42,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:42,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:43:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c200f9a31e804109af18d0e0847c6ee1_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_1/A:col10/1731962622331/Put/seqid=0 2024-11-18T20:43:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742382_1558 (size=12154) 2024-11-18T20:43:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:42,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962682680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962682685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962682685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962682686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962682797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962682797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962682797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962682797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T20:43:42,978 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:42,978 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:43:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-18T20:43:42,979 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:43:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:43:42,981 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:43:42,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:43:43,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962683007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962683007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962683007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962683007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,063 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:43,066 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c200f9a31e804109af18d0e0847c6ee1_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c200f9a31e804109af18d0e0847c6ee1_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:43,067 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/c77fedc983714974a8622cf14c80013a, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:43,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/c77fedc983714974a8622cf14c80013a is 175, key is test_row_1/A:col10/1731962622331/Put/seqid=0 2024-11-18T20:43:43,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742383_1559 (size=30951) 2024-11-18T20:43:43,070 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=124, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/c77fedc983714974a8622cf14c80013a 2024-11-18T20:43:43,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/0f5132ae21164915865fe8cdc1f84913 is 50, key is test_row_1/B:col10/1731962622331/Put/seqid=0 2024-11-18T20:43:43,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742384_1560 (size=9657) 2024-11-18T20:43:43,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/0f5132ae21164915865fe8cdc1f84913 2024-11-18T20:43:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:43:43,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/c75e54db3bc34cbaaed82e90ed6d720f is 50, key is test_row_1/C:col10/1731962622331/Put/seqid=0 2024-11-18T20:43:43,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742385_1561 (size=9657) 2024-11-18T20:43:43,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/c75e54db3bc34cbaaed82e90ed6d720f 2024-11-18T20:43:43,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/c77fedc983714974a8622cf14c80013a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c77fedc983714974a8622cf14c80013a 2024-11-18T20:43:43,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c77fedc983714974a8622cf14c80013a, entries=150, sequenceid=124, filesize=30.2 K 2024-11-18T20:43:43,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/0f5132ae21164915865fe8cdc1f84913 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/0f5132ae21164915865fe8cdc1f84913 2024-11-18T20:43:43,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/0f5132ae21164915865fe8cdc1f84913, entries=100, sequenceid=124, filesize=9.4 K 2024-11-18T20:43:43,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/c75e54db3bc34cbaaed82e90ed6d720f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c75e54db3bc34cbaaed82e90ed6d720f 2024-11-18T20:43:43,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c75e54db3bc34cbaaed82e90ed6d720f, entries=100, sequenceid=124, filesize=9.4 K 2024-11-18T20:43:43,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for db394ce1aa4c72690b92156fd4dc5d2d in 457ms, sequenceid=124, compaction requested=true 2024-11-18T20:43:43,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:43,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:43:43,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:43,109 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:43,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:43:43,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:43,109 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:43,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:43:43,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:43,110 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93067 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:43,110 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:43,110 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/B is initiating minor compaction (all files) 2024-11-18T20:43:43,110 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/A is initiating minor compaction (all files) 2024-11-18T20:43:43,110 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/B in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:43,110 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/A in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:43,110 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/3e8ec9ec41564d8eae67781bc506a96b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/c15212ece9744c56aa60022363c7c0e5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/0f5132ae21164915865fe8cdc1f84913] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=33.1 K 2024-11-18T20:43:43,110 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/3d85e2fc001d49da9af2f5020557b597, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/439df09e7bfc4f068ac1a0e09e88e437, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c77fedc983714974a8622cf14c80013a] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=90.9 K 2024-11-18T20:43:43,110 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:43,110 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/3d85e2fc001d49da9af2f5020557b597, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/439df09e7bfc4f068ac1a0e09e88e437, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c77fedc983714974a8622cf14c80013a] 2024-11-18T20:43:43,110 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e8ec9ec41564d8eae67781bc506a96b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731962621546 2024-11-18T20:43:43,110 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d85e2fc001d49da9af2f5020557b597, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731962621546 2024-11-18T20:43:43,110 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting c15212ece9744c56aa60022363c7c0e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731962621674 2024-11-18T20:43:43,110 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 439df09e7bfc4f068ac1a0e09e88e437, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731962621674 2024-11-18T20:43:43,111 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f5132ae21164915865fe8cdc1f84913, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731962622331 2024-11-18T20:43:43,111 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting c77fedc983714974a8622cf14c80013a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731962622327 2024-11-18T20:43:43,117 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:43,118 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#B#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:43,119 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118b1cd5da43ba94e03aaab82e9906d2019_db394ce1aa4c72690b92156fd4dc5d2d store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:43,119 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/dccf41922d8f4e008d8aec59e7e2acce is 50, key is test_row_0/B:col10/1731962621681/Put/seqid=0 2024-11-18T20:43:43,121 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118b1cd5da43ba94e03aaab82e9906d2019_db394ce1aa4c72690b92156fd4dc5d2d, store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:43,121 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b1cd5da43ba94e03aaab82e9906d2019_db394ce1aa4c72690b92156fd4dc5d2d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:43,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:43:43,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:43,134 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T20:43:43,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:43:43,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:43,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:43:43,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:43,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:43:43,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:43,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742386_1562 (size=12309) 2024-11-18T20:43:43,140 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/dccf41922d8f4e008d8aec59e7e2acce as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/dccf41922d8f4e008d8aec59e7e2acce 2024-11-18T20:43:43,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742387_1563 (size=4469) 2024-11-18T20:43:43,148 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/B of db394ce1aa4c72690b92156fd4dc5d2d into dccf41922d8f4e008d8aec59e7e2acce(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:43,148 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:43,148 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/B, priority=13, startTime=1731962623109; duration=0sec 2024-11-18T20:43:43,148 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:43,148 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:B 2024-11-18T20:43:43,148 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:43,149 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#A#compaction#480 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:43,149 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/6966e2f518fa4667b10350c8e5cc1ab7 is 175, key is test_row_0/A:col10/1731962621681/Put/seqid=0 2024-11-18T20:43:43,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d5211b8f894a420fa22bdc241e13b005_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_0/A:col10/1731962622684/Put/seqid=0 2024-11-18T20:43:43,152 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:43,152 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/C is initiating minor compaction (all files) 2024-11-18T20:43:43,152 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/C in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:43,152 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c2b4225f3d1f4362b607b78c4bb2c110, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/744c80bd0a3d4e199401f442e1a032b8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c75e54db3bc34cbaaed82e90ed6d720f] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=33.1 K 2024-11-18T20:43:43,154 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting c2b4225f3d1f4362b607b78c4bb2c110, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731962621546 2024-11-18T20:43:43,155 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 744c80bd0a3d4e199401f442e1a032b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731962621674 2024-11-18T20:43:43,155 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting c75e54db3bc34cbaaed82e90ed6d720f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731962622331 2024-11-18T20:43:43,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742388_1564 (size=31370) 2024-11-18T20:43:43,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742389_1565 (size=12304) 2024-11-18T20:43:43,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:43,171 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#C#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:43,171 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d5211b8f894a420fa22bdc241e13b005_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d5211b8f894a420fa22bdc241e13b005_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:43,171 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/15aa6fd697f94db18092069c3a089a62 is 50, key is test_row_0/C:col10/1731962621681/Put/seqid=0 2024-11-18T20:43:43,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/099c46959e084b158fec4c4b19f0b89b, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:43,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/099c46959e084b158fec4c4b19f0b89b is 175, key is test_row_0/A:col10/1731962622684/Put/seqid=0 2024-11-18T20:43:43,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742391_1567 (size=31105) 2024-11-18T20:43:43,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742390_1566 (size=12309) 2024-11-18T20:43:43,191 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/15aa6fd697f94db18092069c3a089a62 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/15aa6fd697f94db18092069c3a089a62 2024-11-18T20:43:43,197 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/C of db394ce1aa4c72690b92156fd4dc5d2d into 15aa6fd697f94db18092069c3a089a62(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:43,197 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:43,197 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/C, priority=13, startTime=1731962623109; duration=0sec 2024-11-18T20:43:43,197 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:43,197 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:C 2024-11-18T20:43:43,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:43:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:43,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:43,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962683335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962683335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962683336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962683340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962683447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962683447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962683447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962683448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,562 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/6966e2f518fa4667b10350c8e5cc1ab7 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6966e2f518fa4667b10350c8e5cc1ab7 2024-11-18T20:43:43,566 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/A of db394ce1aa4c72690b92156fd4dc5d2d into 6966e2f518fa4667b10350c8e5cc1ab7(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:43,566 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:43,566 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/A, priority=13, startTime=1731962623109; duration=0sec 2024-11-18T20:43:43,566 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:43,566 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:A 2024-11-18T20:43:43,583 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/099c46959e084b158fec4c4b19f0b89b 2024-11-18T20:43:43,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/ea6fbad15ed74d7dac2422f04f65d05d is 50, key is test_row_0/B:col10/1731962622684/Put/seqid=0 2024-11-18T20:43:43,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742392_1568 (size=12151) 2024-11-18T20:43:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:43:43,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962683657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962683657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962683657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962683658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962683967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962683967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962683967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962683967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:44,001 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/ea6fbad15ed74d7dac2422f04f65d05d 2024-11-18T20:43:44,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/ed59148c1a474781ac39cd8004931833 is 50, key is test_row_0/C:col10/1731962622684/Put/seqid=0 2024-11-18T20:43:44,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742393_1569 (size=12151) 2024-11-18T20:43:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:43:44,423 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/ed59148c1a474781ac39cd8004931833 2024-11-18T20:43:44,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/099c46959e084b158fec4c4b19f0b89b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/099c46959e084b158fec4c4b19f0b89b 2024-11-18T20:43:44,429 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/099c46959e084b158fec4c4b19f0b89b, entries=150, sequenceid=139, filesize=30.4 K 2024-11-18T20:43:44,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/ea6fbad15ed74d7dac2422f04f65d05d as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/ea6fbad15ed74d7dac2422f04f65d05d 2024-11-18T20:43:44,432 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/ea6fbad15ed74d7dac2422f04f65d05d, entries=150, sequenceid=139, filesize=11.9 K 2024-11-18T20:43:44,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/ed59148c1a474781ac39cd8004931833 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/ed59148c1a474781ac39cd8004931833 2024-11-18T20:43:44,437 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/ed59148c1a474781ac39cd8004931833, entries=150, sequenceid=139, filesize=11.9 K 2024-11-18T20:43:44,438 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for db394ce1aa4c72690b92156fd4dc5d2d in 1305ms, sequenceid=139, compaction requested=false 2024-11-18T20:43:44,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:44,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:44,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-18T20:43:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-18T20:43:44,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-18T20:43:44,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4580 sec 2024-11-18T20:43:44,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4620 sec 2024-11-18T20:43:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:44,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T20:43:44,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:43:44,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:44,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:43:44,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:44,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:43:44,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:44,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118218bd752a1364005b9ff6a6917b4f274_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_0/A:col10/1731962623335/Put/seqid=0 2024-11-18T20:43:44,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742394_1570 (size=14794) 2024-11-18T20:43:44,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962684490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:44,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:44,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962684493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:44,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:44,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962684495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:44,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:44,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962684495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:44,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962684607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:44,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962684608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:44,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962684608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:44,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962684608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:44,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:44,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:44,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:44,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35952 deadline: 1731962684777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:44,781 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in db394ce1aa4c72690b92156fd4dc5d2d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:44,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:44,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962684817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:44,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962684818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:44,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962684818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:44,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962684818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:44,892 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:44,894 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118218bd752a1364005b9ff6a6917b4f274_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118218bd752a1364005b9ff6a6917b4f274_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:44,895 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/3eeee260dfd84c02b0e6947ed1d5191f, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:44,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/3eeee260dfd84c02b0e6947ed1d5191f is 175, key is test_row_0/A:col10/1731962623335/Put/seqid=0 2024-11-18T20:43:44,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742395_1571 (size=39749) 2024-11-18T20:43:45,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:43:45,127 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:45,128 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:43:45,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-18T20:43:45,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T20:43:45,129 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:43:45,130 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:43:45,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:43:45,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:45,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:45,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962685127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:45,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962685127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962685127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:45,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962685127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:45,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T20:43:45,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T20:43:45,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:45,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:45,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:45,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:45,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:45,299 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/3eeee260dfd84c02b0e6947ed1d5191f 2024-11-18T20:43:45,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/e2524d8adf7a4deaaeda4806ef0a4fe4 is 50, key is test_row_0/B:col10/1731962623335/Put/seqid=0 2024-11-18T20:43:45,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742396_1572 (size=12151) 2024-11-18T20:43:45,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/e2524d8adf7a4deaaeda4806ef0a4fe4 2024-11-18T20:43:45,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/8dc1fdfa4bc34150a5d48a39849a796d is 50, key is test_row_0/C:col10/1731962623335/Put/seqid=0 2024-11-18T20:43:45,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742397_1573 (size=12151) 2024-11-18T20:43:45,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T20:43:45,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:45,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:45,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:45,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:45,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T20:43:45,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T20:43:45,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:45,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:45,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:45,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:45,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:45,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962685637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:45,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962685637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:45,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962685637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:45,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962685647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:45,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/8dc1fdfa4bc34150a5d48a39849a796d 2024-11-18T20:43:45,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/3eeee260dfd84c02b0e6947ed1d5191f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/3eeee260dfd84c02b0e6947ed1d5191f 2024-11-18T20:43:45,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/3eeee260dfd84c02b0e6947ed1d5191f, entries=200, sequenceid=160, filesize=38.8 K 2024-11-18T20:43:45,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/e2524d8adf7a4deaaeda4806ef0a4fe4 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/e2524d8adf7a4deaaeda4806ef0a4fe4 2024-11-18T20:43:45,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/e2524d8adf7a4deaaeda4806ef0a4fe4, entries=150, sequenceid=160, filesize=11.9 K 2024-11-18T20:43:45,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/8dc1fdfa4bc34150a5d48a39849a796d as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/8dc1fdfa4bc34150a5d48a39849a796d 2024-11-18T20:43:45,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/8dc1fdfa4bc34150a5d48a39849a796d, entries=150, sequenceid=160, filesize=11.9 K 2024-11-18T20:43:45,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T20:43:45,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for db394ce1aa4c72690b92156fd4dc5d2d in 1262ms, sequenceid=160, compaction requested=true 2024-11-18T20:43:45,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:45,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:45,741 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T20:43:45,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:43:45,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:45,741 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:45,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:43:45,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:43:45,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:45,741 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:45,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:43:45,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:45,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:45,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:43:45,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:45,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:43:45,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:45,742 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102224 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:45,742 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:45,742 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/A is initiating minor compaction (all files) 2024-11-18T20:43:45,742 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/B is initiating minor compaction (all files) 2024-11-18T20:43:45,742 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/A in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:45,742 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/B in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:45,742 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6966e2f518fa4667b10350c8e5cc1ab7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/099c46959e084b158fec4c4b19f0b89b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/3eeee260dfd84c02b0e6947ed1d5191f] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=99.8 K 2024-11-18T20:43:45,742 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/dccf41922d8f4e008d8aec59e7e2acce, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/ea6fbad15ed74d7dac2422f04f65d05d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/e2524d8adf7a4deaaeda4806ef0a4fe4] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=35.8 K 2024-11-18T20:43:45,742 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:45,742 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6966e2f518fa4667b10350c8e5cc1ab7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/099c46959e084b158fec4c4b19f0b89b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/3eeee260dfd84c02b0e6947ed1d5191f] 2024-11-18T20:43:45,743 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting dccf41922d8f4e008d8aec59e7e2acce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731962621681 2024-11-18T20:43:45,744 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting ea6fbad15ed74d7dac2422f04f65d05d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731962622675 2024-11-18T20:43:45,744 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting e2524d8adf7a4deaaeda4806ef0a4fe4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731962623335 2024-11-18T20:43:45,744 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6966e2f518fa4667b10350c8e5cc1ab7, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731962621681 2024-11-18T20:43:45,745 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 099c46959e084b158fec4c4b19f0b89b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731962622675 2024-11-18T20:43:45,745 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eeee260dfd84c02b0e6947ed1d5191f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731962623333 2024-11-18T20:43:45,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111837b0d1d5bc1f4de8abf4238147c24e25_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_0/A:col10/1731962624494/Put/seqid=0 2024-11-18T20:43:45,750 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#B#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:45,751 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/9f59697215374e058e1154e99d942cb4 is 50, key is test_row_0/B:col10/1731962623335/Put/seqid=0 2024-11-18T20:43:45,752 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T20:43:45,766 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111836685ab4a4af44e38c1a74cbf1cce67f_db394ce1aa4c72690b92156fd4dc5d2d store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:45,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742399_1575 (size=12561) 2024-11-18T20:43:45,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742398_1574 (size=12304) 2024-11-18T20:43:45,768 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111836685ab4a4af44e38c1a74cbf1cce67f_db394ce1aa4c72690b92156fd4dc5d2d, store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:45,768 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111836685ab4a4af44e38c1a74cbf1cce67f_db394ce1aa4c72690b92156fd4dc5d2d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:45,772 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/9f59697215374e058e1154e99d942cb4 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/9f59697215374e058e1154e99d942cb4 2024-11-18T20:43:45,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742400_1576 (size=4469) 2024-11-18T20:43:45,776 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/B of db394ce1aa4c72690b92156fd4dc5d2d into 9f59697215374e058e1154e99d942cb4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:45,776 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:45,776 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/B, priority=13, startTime=1731962625741; duration=0sec 2024-11-18T20:43:45,776 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:45,776 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:B 2024-11-18T20:43:45,776 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:45,777 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:45,777 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/C is initiating minor compaction (all files) 2024-11-18T20:43:45,777 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/C in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:45,777 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/15aa6fd697f94db18092069c3a089a62, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/ed59148c1a474781ac39cd8004931833, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/8dc1fdfa4bc34150a5d48a39849a796d] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=35.8 K 2024-11-18T20:43:45,778 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 15aa6fd697f94db18092069c3a089a62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731962621681 2024-11-18T20:43:45,778 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting ed59148c1a474781ac39cd8004931833, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731962622675 2024-11-18T20:43:45,778 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dc1fdfa4bc34150a5d48a39849a796d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731962623335 2024-11-18T20:43:45,783 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#C#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:45,783 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/05565d2248174e4fa5de48d59f0d3c59 is 50, key is test_row_0/C:col10/1731962623335/Put/seqid=0 2024-11-18T20:43:45,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742401_1577 (size=12561) 2024-11-18T20:43:45,789 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/05565d2248174e4fa5de48d59f0d3c59 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/05565d2248174e4fa5de48d59f0d3c59 2024-11-18T20:43:45,793 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/C of db394ce1aa4c72690b92156fd4dc5d2d into 05565d2248174e4fa5de48d59f0d3c59(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:45,793 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:45,793 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/C, priority=13, startTime=1731962625741; duration=0sec 2024-11-18T20:43:45,793 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:45,793 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:C 2024-11-18T20:43:46,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,171 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111837b0d1d5bc1f4de8abf4238147c24e25_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111837b0d1d5bc1f4de8abf4238147c24e25_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:46,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/8ad2b2276fb54cee9a4ca266624ca338, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:46,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/8ad2b2276fb54cee9a4ca266624ca338 is 175, key is test_row_0/A:col10/1731962624494/Put/seqid=0 2024-11-18T20:43:46,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742402_1578 (size=31105) 2024-11-18T20:43:46,175 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#A#compaction#490 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:46,176 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/137389d95d424f618f2620bf9aa7bce8 is 175, key is test_row_0/A:col10/1731962623335/Put/seqid=0 2024-11-18T20:43:46,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742403_1579 (size=31515) 2024-11-18T20:43:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T20:43:46,575 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/8ad2b2276fb54cee9a4ca266624ca338 2024-11-18T20:43:46,583 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/137389d95d424f618f2620bf9aa7bce8 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/137389d95d424f618f2620bf9aa7bce8 2024-11-18T20:43:46,588 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/A of db394ce1aa4c72690b92156fd4dc5d2d into 137389d95d424f618f2620bf9aa7bce8(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:46,588 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:46,588 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/A, priority=13, startTime=1731962625741; duration=0sec 2024-11-18T20:43:46,588 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:46,588 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:A 2024-11-18T20:43:46,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/3a91c2f23a714184bfa616bcb2577fcb is 50, key is test_row_0/B:col10/1731962624494/Put/seqid=0 2024-11-18T20:43:46,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742404_1580 (size=12151) 2024-11-18T20:43:46,599 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/3a91c2f23a714184bfa616bcb2577fcb 2024-11-18T20:43:46,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/78508445e18046a8af40c6aede6930ca is 50, key is test_row_0/C:col10/1731962624494/Put/seqid=0 2024-11-18T20:43:46,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742405_1581 (size=12151) 2024-11-18T20:43:46,634 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/78508445e18046a8af40c6aede6930ca 2024-11-18T20:43:46,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/8ad2b2276fb54cee9a4ca266624ca338 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/8ad2b2276fb54cee9a4ca266624ca338 2024-11-18T20:43:46,645 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/8ad2b2276fb54cee9a4ca266624ca338, entries=150, sequenceid=174, filesize=30.4 K 2024-11-18T20:43:46,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/3a91c2f23a714184bfa616bcb2577fcb as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/3a91c2f23a714184bfa616bcb2577fcb 2024-11-18T20:43:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,650 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/3a91c2f23a714184bfa616bcb2577fcb, entries=150, sequenceid=174, filesize=11.9 K 2024-11-18T20:43:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/78508445e18046a8af40c6aede6930ca as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/78508445e18046a8af40c6aede6930ca 2024-11-18T20:43:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,654 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/78508445e18046a8af40c6aede6930ca, entries=150, sequenceid=174, filesize=11.9 K 2024-11-18T20:43:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,656 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for db394ce1aa4c72690b92156fd4dc5d2d in 915ms, sequenceid=174, compaction requested=false 2024-11-18T20:43:46,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:46,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:46,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-18T20:43:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-18T20:43:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-18T20:43:46,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5280 sec 2024-11-18T20:43:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.5330 sec 2024-11-18T20:43:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T20:43:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:43:46,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:46,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:43:46,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:46,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:43:46,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:46,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:46,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184a63e53b38774f7ba84b43b8102efe2c_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_0/A:col10/1731962626694/Put/seqid=0 2024-11-18T20:43:46,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742406_1582 (size=24758) 2024-11-18T20:43:46,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:46,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962686733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962686734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:46,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731962626710, max timestamp=1731962626739Num uniques 859; 2024-11-18T20:43:46,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731962626710, max timestamp=1731962626739Num uniques 859; 2024-11-18T20:43:46,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:43:46,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:43:46,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731962626710, max timestamp=1731962626739Num uniques 859; 2024-11-18T20:43:46,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c0a89b2656d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T20:43:46,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962686794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:46,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962686794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:46,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:46,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962686847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:46,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962686848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:46,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962686908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:46,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962686908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:47,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:47,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962687057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:47,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:47,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962687057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:47,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:47,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962687117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:47,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962687117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:47,131 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:47,133 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184a63e53b38774f7ba84b43b8102efe2c_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184a63e53b38774f7ba84b43b8102efe2c_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:47,134 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/ee15a28f0e954f30b79f66bbcfaeaaba, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:47,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/ee15a28f0e954f30b79f66bbcfaeaaba is 175, key is test_row_0/A:col10/1731962626694/Put/seqid=0 2024-11-18T20:43:47,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742407_1583 (size=74395) 2024-11-18T20:43:47,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T20:43:47,278 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:47,278 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:43:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-18T20:43:47,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:43:47,281 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:43:47,281 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:43:47,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:43:47,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962687367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:47,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962687367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:43:47,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962687427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962687427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:47,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T20:43:47,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:47,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:47,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:47,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:47,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:47,537 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/ee15a28f0e954f30b79f66bbcfaeaaba 2024-11-18T20:43:47,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/b64d96ae03914cf9b07ddebd7c174ec2 is 50, key is test_row_0/B:col10/1731962626694/Put/seqid=0 2024-11-18T20:43:47,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742408_1584 (size=12151) 2024-11-18T20:43:47,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/b64d96ae03914cf9b07ddebd7c174ec2 2024-11-18T20:43:47,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/4422348d1e464368a925afaf4f1a1243 is 50, key is test_row_0/C:col10/1731962626694/Put/seqid=0 2024-11-18T20:43:47,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742409_1585 (size=12151) 2024-11-18T20:43:47,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/4422348d1e464368a925afaf4f1a1243 2024-11-18T20:43:47,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/ee15a28f0e954f30b79f66bbcfaeaaba as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/ee15a28f0e954f30b79f66bbcfaeaaba 2024-11-18T20:43:47,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/ee15a28f0e954f30b79f66bbcfaeaaba, entries=400, sequenceid=188, filesize=72.7 K 2024-11-18T20:43:47,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/b64d96ae03914cf9b07ddebd7c174ec2 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/b64d96ae03914cf9b07ddebd7c174ec2 2024-11-18T20:43:47,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/b64d96ae03914cf9b07ddebd7c174ec2, entries=150, sequenceid=188, filesize=11.9 K 2024-11-18T20:43:47,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/4422348d1e464368a925afaf4f1a1243 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/4422348d1e464368a925afaf4f1a1243 2024-11-18T20:43:47,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/4422348d1e464368a925afaf4f1a1243, entries=150, sequenceid=188, filesize=11.9 K 2024-11-18T20:43:47,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for db394ce1aa4c72690b92156fd4dc5d2d in 886ms, sequenceid=188, compaction requested=true 2024-11-18T20:43:47,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:47,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:43:47,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:47,580 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:47,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:43:47,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:47,580 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:47,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:43:47,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:47,582 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:47,582 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/B is initiating minor compaction (all files) 2024-11-18T20:43:47,582 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/B in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:47,582 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/9f59697215374e058e1154e99d942cb4, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/3a91c2f23a714184bfa616bcb2577fcb, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/b64d96ae03914cf9b07ddebd7c174ec2] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=36.0 K 2024-11-18T20:43:47,582 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f59697215374e058e1154e99d942cb4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731962623335 2024-11-18T20:43:47,582 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 137015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:47,582 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/A is initiating minor compaction (all files) 2024-11-18T20:43:47,582 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/A in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:47,582 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/137389d95d424f618f2620bf9aa7bce8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/8ad2b2276fb54cee9a4ca266624ca338, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/ee15a28f0e954f30b79f66bbcfaeaaba] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=133.8 K 2024-11-18T20:43:47,582 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:47,582 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a91c2f23a714184bfa616bcb2577fcb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731962624491 2024-11-18T20:43:47,582 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/137389d95d424f618f2620bf9aa7bce8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/8ad2b2276fb54cee9a4ca266624ca338, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/ee15a28f0e954f30b79f66bbcfaeaaba] 2024-11-18T20:43:47,583 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting b64d96ae03914cf9b07ddebd7c174ec2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731962626687 2024-11-18T20:43:47,584 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 137389d95d424f618f2620bf9aa7bce8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731962623335 2024-11-18T20:43:47,584 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ad2b2276fb54cee9a4ca266624ca338, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731962624491 2024-11-18T20:43:47,585 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee15a28f0e954f30b79f66bbcfaeaaba, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731962626676 2024-11-18T20:43:47,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T20:43:47,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:47,586 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-18T20:43:47,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:43:47,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:47,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:43:47,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:47,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:43:47,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T20:43:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:43:47,602 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:47,603 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#B#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:47,604 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/d2a6f8bfe5b54c728a933b5eb23b5641 is 50, key is test_row_0/B:col10/1731962626694/Put/seqid=0 2024-11-18T20:43:47,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ce3e285d6d7247739319ff53429ce07b_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_0/A:col10/1731962626793/Put/seqid=0 2024-11-18T20:43:47,610 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411183f3108be2419471789a955a03086d618_db394ce1aa4c72690b92156fd4dc5d2d store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:47,613 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411183f3108be2419471789a955a03086d618_db394ce1aa4c72690b92156fd4dc5d2d, store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:47,613 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183f3108be2419471789a955a03086d618_db394ce1aa4c72690b92156fd4dc5d2d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:47,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742410_1586 (size=12304) 2024-11-18T20:43:47,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742411_1587 (size=12663) 2024-11-18T20:43:47,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742412_1588 (size=4469) 2024-11-18T20:43:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:47,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:47,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962687898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:47,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:47,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962687901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:47,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:43:47,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962687937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:47,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962687947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:48,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962688007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:48,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:48,019 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ce3e285d6d7247739319ff53429ce07b_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ce3e285d6d7247739319ff53429ce07b_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:48,019 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/d2a6f8bfe5b54c728a933b5eb23b5641 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/d2a6f8bfe5b54c728a933b5eb23b5641 2024-11-18T20:43:48,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/91291f8452844b2996af2733457189cc, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:48,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/91291f8452844b2996af2733457189cc is 175, key is test_row_0/A:col10/1731962626793/Put/seqid=0 2024-11-18T20:43:48,020 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#A#compaction#498 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:48,020 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/4d334d05ce1741c59693ec1633ba2692 is 175, key is test_row_0/A:col10/1731962626694/Put/seqid=0 2024-11-18T20:43:48,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962688017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:48,024 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/B of db394ce1aa4c72690b92156fd4dc5d2d into d2a6f8bfe5b54c728a933b5eb23b5641(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:48,024 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:48,024 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/B, priority=13, startTime=1731962627580; duration=0sec 2024-11-18T20:43:48,024 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:48,024 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:B 2024-11-18T20:43:48,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742413_1589 (size=31105) 2024-11-18T20:43:48,024 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:48,025 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/91291f8452844b2996af2733457189cc 2024-11-18T20:43:48,025 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:48,025 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/C is initiating minor compaction (all files) 2024-11-18T20:43:48,025 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/C in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:48,025 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/05565d2248174e4fa5de48d59f0d3c59, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/78508445e18046a8af40c6aede6930ca, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/4422348d1e464368a925afaf4f1a1243] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=36.0 K 2024-11-18T20:43:48,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742414_1590 (size=31617) 2024-11-18T20:43:48,026 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 05565d2248174e4fa5de48d59f0d3c59, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731962623335 2024-11-18T20:43:48,026 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 78508445e18046a8af40c6aede6930ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731962624491 2024-11-18T20:43:48,026 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4422348d1e464368a925afaf4f1a1243, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731962626687 2024-11-18T20:43:48,031 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/4d334d05ce1741c59693ec1633ba2692 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/4d334d05ce1741c59693ec1633ba2692 2024-11-18T20:43:48,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/fd330a522751423f8877b10d69e8e66e is 50, key is test_row_0/B:col10/1731962626793/Put/seqid=0 2024-11-18T20:43:48,037 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/A of db394ce1aa4c72690b92156fd4dc5d2d into 4d334d05ce1741c59693ec1633ba2692(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:48,037 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:48,037 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/A, priority=13, startTime=1731962627580; duration=0sec 2024-11-18T20:43:48,037 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:48,037 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:A 2024-11-18T20:43:48,043 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#C#compaction#501 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:48,043 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/3e1c762614824236a05b7b4a47cc39d0 is 50, key is test_row_0/C:col10/1731962626694/Put/seqid=0 2024-11-18T20:43:48,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742415_1591 (size=12151) 2024-11-18T20:43:48,059 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/fd330a522751423f8877b10d69e8e66e 2024-11-18T20:43:48,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742416_1592 (size=12663) 2024-11-18T20:43:48,065 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/3e1c762614824236a05b7b4a47cc39d0 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/3e1c762614824236a05b7b4a47cc39d0 2024-11-18T20:43:48,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/7a9ebe89b2d54030a665818339a67582 is 50, key is test_row_0/C:col10/1731962626793/Put/seqid=0 2024-11-18T20:43:48,072 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/C of db394ce1aa4c72690b92156fd4dc5d2d into 3e1c762614824236a05b7b4a47cc39d0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:48,072 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:48,072 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/C, priority=13, startTime=1731962627580; duration=0sec 2024-11-18T20:43:48,072 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:48,072 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:C 2024-11-18T20:43:48,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742417_1593 (size=12151) 2024-11-18T20:43:48,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962688217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:48,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962688227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:48,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:43:48,479 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/7a9ebe89b2d54030a665818339a67582 2024-11-18T20:43:48,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/91291f8452844b2996af2733457189cc as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/91291f8452844b2996af2733457189cc 2024-11-18T20:43:48,485 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/91291f8452844b2996af2733457189cc, entries=150, sequenceid=223, filesize=30.4 K 2024-11-18T20:43:48,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/fd330a522751423f8877b10d69e8e66e as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/fd330a522751423f8877b10d69e8e66e 2024-11-18T20:43:48,490 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/fd330a522751423f8877b10d69e8e66e, entries=150, sequenceid=223, filesize=11.9 K 2024-11-18T20:43:48,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/7a9ebe89b2d54030a665818339a67582 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/7a9ebe89b2d54030a665818339a67582 2024-11-18T20:43:48,495 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/7a9ebe89b2d54030a665818339a67582, entries=150, sequenceid=223, filesize=11.9 K 2024-11-18T20:43:48,496 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for db394ce1aa4c72690b92156fd4dc5d2d in 910ms, sequenceid=223, compaction requested=false 2024-11-18T20:43:48,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:48,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:48,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-18T20:43:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-18T20:43:48,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-18T20:43:48,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2160 sec 2024-11-18T20:43:48,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.2210 sec 2024-11-18T20:43:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:48,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T20:43:48,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:43:48,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:48,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:43:48,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:48,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:43:48,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:48,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118cc05e49cd62b4f2fb76aebfc088b6baf_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_0/A:col10/1731962627900/Put/seqid=0 2024-11-18T20:43:48,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742418_1594 (size=14794) 2024-11-18T20:43:48,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962688579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:48,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962688580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:48,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962688697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962688697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:48,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:48,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962688907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:48,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:48,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962688907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:48,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:48,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:48,940 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:48,944 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118cc05e49cd62b4f2fb76aebfc088b6baf_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118cc05e49cd62b4f2fb76aebfc088b6baf_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:48,945 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/8c9cd6649ca041c99d3bfb7986c077e0, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:48,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/8c9cd6649ca041c99d3bfb7986c077e0 is 175, key is test_row_0/A:col10/1731962627900/Put/seqid=0 2024-11-18T20:43:48,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962688947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:48,949 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in db394ce1aa4c72690b92156fd4dc5d2d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:48,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:48,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742419_1595 (size=39749) 2024-11-18T20:43:48,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962688957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:48,961 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in db394ce1aa4c72690b92156fd4dc5d2d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:49,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962689217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:49,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962689217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:49,354 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/8c9cd6649ca041c99d3bfb7986c077e0 2024-11-18T20:43:49,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/4fd0e383f92f4789877c776e12bba4ce is 50, key is test_row_0/B:col10/1731962627900/Put/seqid=0 2024-11-18T20:43:49,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742420_1596 (size=12151) 2024-11-18T20:43:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:43:49,428 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:49,428 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:43:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-18T20:43:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:43:49,430 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:43:49,431 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:43:49,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:43:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:43:49,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T20:43:49,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:49,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:49,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:49,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:49,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:49,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962689732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:49,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T20:43:49,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962689732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:49,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:49,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:49,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:49,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:49,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:49,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:49,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:43:49,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/4fd0e383f92f4789877c776e12bba4ce 2024-11-18T20:43:49,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/98649dda6d334903a2b987d8c70cb92b is 50, key is test_row_0/C:col10/1731962627900/Put/seqid=0 2024-11-18T20:43:49,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742421_1597 (size=12151) 2024-11-18T20:43:49,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T20:43:49,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:49,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:49,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:49,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:49,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:50,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T20:43:50,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:50,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:50,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:50,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:50,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:43:50,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/98649dda6d334903a2b987d8c70cb92b 2024-11-18T20:43:50,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/8c9cd6649ca041c99d3bfb7986c077e0 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/8c9cd6649ca041c99d3bfb7986c077e0 2024-11-18T20:43:50,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/8c9cd6649ca041c99d3bfb7986c077e0, entries=200, sequenceid=238, filesize=38.8 K 2024-11-18T20:43:50,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/4fd0e383f92f4789877c776e12bba4ce as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/4fd0e383f92f4789877c776e12bba4ce 2024-11-18T20:43:50,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T20:43:50,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:50,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:50,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:50,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:50,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/4fd0e383f92f4789877c776e12bba4ce, entries=150, sequenceid=238, filesize=11.9 K 2024-11-18T20:43:50,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:50,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/98649dda6d334903a2b987d8c70cb92b as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/98649dda6d334903a2b987d8c70cb92b 2024-11-18T20:43:50,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/98649dda6d334903a2b987d8c70cb92b, entries=150, sequenceid=238, filesize=11.9 K 2024-11-18T20:43:50,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for db394ce1aa4c72690b92156fd4dc5d2d in 1675ms, sequenceid=238, compaction requested=true 2024-11-18T20:43:50,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:50,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:43:50,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:50,206 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:50,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:43:50,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:50,206 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:50,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:43:50,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:50,206 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:50,206 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:50,206 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/B is initiating minor compaction (all files) 2024-11-18T20:43:50,206 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/A is initiating minor compaction (all files) 2024-11-18T20:43:50,207 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/B in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:50,207 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/A in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:50,207 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/d2a6f8bfe5b54c728a933b5eb23b5641, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/fd330a522751423f8877b10d69e8e66e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/4fd0e383f92f4789877c776e12bba4ce] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=36.1 K 2024-11-18T20:43:50,207 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/4d334d05ce1741c59693ec1633ba2692, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/91291f8452844b2996af2733457189cc, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/8c9cd6649ca041c99d3bfb7986c077e0] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=100.1 K 2024-11-18T20:43:50,207 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:50,207 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/4d334d05ce1741c59693ec1633ba2692, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/91291f8452844b2996af2733457189cc, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/8c9cd6649ca041c99d3bfb7986c077e0] 2024-11-18T20:43:50,207 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting d2a6f8bfe5b54c728a933b5eb23b5641, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731962626687 2024-11-18T20:43:50,207 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d334d05ce1741c59693ec1633ba2692, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731962626687 2024-11-18T20:43:50,208 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting fd330a522751423f8877b10d69e8e66e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731962626769 2024-11-18T20:43:50,208 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91291f8452844b2996af2733457189cc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731962626769 2024-11-18T20:43:50,209 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fd0e383f92f4789877c776e12bba4ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731962627886 2024-11-18T20:43:50,209 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c9cd6649ca041c99d3bfb7986c077e0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731962627886 2024-11-18T20:43:50,213 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:50,214 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411187ea73463d96c45e09258e40394a968fa_db394ce1aa4c72690b92156fd4dc5d2d store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:50,215 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#B#compaction#507 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:50,215 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/1b655a8adee24a579eaccd4abb9aba84 is 50, key is test_row_0/B:col10/1731962627900/Put/seqid=0 2024-11-18T20:43:50,217 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411187ea73463d96c45e09258e40394a968fa_db394ce1aa4c72690b92156fd4dc5d2d, store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:50,217 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187ea73463d96c45e09258e40394a968fa_db394ce1aa4c72690b92156fd4dc5d2d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:50,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742422_1598 (size=12765) 2024-11-18T20:43:50,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742423_1599 (size=4469) 2024-11-18T20:43:50,237 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#A#compaction#506 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:50,238 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/6fabfe044cbf4ccc806e69fc1540e0e1 is 175, key is test_row_0/A:col10/1731962627900/Put/seqid=0 2024-11-18T20:43:50,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742424_1600 (size=31719) 2024-11-18T20:43:50,268 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/6fabfe044cbf4ccc806e69fc1540e0e1 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6fabfe044cbf4ccc806e69fc1540e0e1 2024-11-18T20:43:50,273 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/A of db394ce1aa4c72690b92156fd4dc5d2d into 6fabfe044cbf4ccc806e69fc1540e0e1(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:50,273 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:50,273 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/A, priority=13, startTime=1731962630206; duration=0sec 2024-11-18T20:43:50,273 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:50,273 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:A 2024-11-18T20:43:50,273 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:50,274 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:50,274 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/C is initiating minor compaction (all files) 2024-11-18T20:43:50,274 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/C in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:50,274 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/3e1c762614824236a05b7b4a47cc39d0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/7a9ebe89b2d54030a665818339a67582, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/98649dda6d334903a2b987d8c70cb92b] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=36.1 K 2024-11-18T20:43:50,276 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e1c762614824236a05b7b4a47cc39d0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731962626687 2024-11-18T20:43:50,276 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a9ebe89b2d54030a665818339a67582, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731962626769 2024-11-18T20:43:50,277 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98649dda6d334903a2b987d8c70cb92b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731962627886 2024-11-18T20:43:50,283 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#C#compaction#508 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:50,283 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/d1c9a213e50846be8dae30aa027ee525 is 50, key is test_row_0/C:col10/1731962627900/Put/seqid=0 2024-11-18T20:43:50,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742425_1601 (size=12765) 2024-11-18T20:43:50,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T20:43:50,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:50,350 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T20:43:50,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:43:50,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:50,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:43:50,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:50,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:43:50,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:50,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118365c2518aeda47b5ae0d8bb7afd78360_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_0/A:col10/1731962628569/Put/seqid=0 2024-11-18T20:43:50,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742426_1602 (size=12354) 2024-11-18T20:43:50,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:43:50,630 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/1b655a8adee24a579eaccd4abb9aba84 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/1b655a8adee24a579eaccd4abb9aba84 2024-11-18T20:43:50,635 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/B of db394ce1aa4c72690b92156fd4dc5d2d into 1b655a8adee24a579eaccd4abb9aba84(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:50,635 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:50,635 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/B, priority=13, startTime=1731962630206; duration=0sec 2024-11-18T20:43:50,635 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:50,635 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:B 2024-11-18T20:43:50,691 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/d1c9a213e50846be8dae30aa027ee525 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/d1c9a213e50846be8dae30aa027ee525 2024-11-18T20:43:50,695 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/C of db394ce1aa4c72690b92156fd4dc5d2d into d1c9a213e50846be8dae30aa027ee525(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:50,695 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:50,695 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/C, priority=13, startTime=1731962630206; duration=0sec 2024-11-18T20:43:50,695 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:50,695 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:C 2024-11-18T20:43:50,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:50,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:50,763 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118365c2518aeda47b5ae0d8bb7afd78360_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118365c2518aeda47b5ae0d8bb7afd78360_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:50,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/da018d8fd3d24466934374ed3811b160, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:50,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/da018d8fd3d24466934374ed3811b160 is 175, key is test_row_0/A:col10/1731962628569/Put/seqid=0 2024-11-18T20:43:50,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:50,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742427_1603 (size=31155) 2024-11-18T20:43:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962690763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:50,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962690765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:50,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962690877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:50,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:50,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962690877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:50,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:50,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:50,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:50,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962690958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:50,962 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in db394ce1aa4c72690b92156fd4dc5d2d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:50,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:50,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:50,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:50,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:50,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962690987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:50,989 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in db394ce1aa4c72690b92156fd4dc5d2d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:51,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:51,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962691088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962691087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:51,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:51,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:51,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:51,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:51,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:51,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:51,168 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/da018d8fd3d24466934374ed3811b160 2024-11-18T20:43:51,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/ca8523efac8d491abf58a161efe795e6 is 50, key is test_row_0/B:col10/1731962628569/Put/seqid=0 2024-11-18T20:43:51,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742428_1604 (size=12201) 2024-11-18T20:43:51,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962691397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:51,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:51,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:51,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:51,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962691397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:51,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:51,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:51,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:43:51,578 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/ca8523efac8d491abf58a161efe795e6 2024-11-18T20:43:51,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/81e3729f695f48d680d7539b17a0e5aa is 50, key is test_row_0/C:col10/1731962628569/Put/seqid=0 2024-11-18T20:43:51,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742429_1605 (size=12201) 2024-11-18T20:43:51,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962691907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:51,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962691907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:51,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:51,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:51,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:51,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:51,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:51,989 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/81e3729f695f48d680d7539b17a0e5aa 2024-11-18T20:43:51,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/da018d8fd3d24466934374ed3811b160 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/da018d8fd3d24466934374ed3811b160 2024-11-18T20:43:51,996 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/da018d8fd3d24466934374ed3811b160, entries=150, sequenceid=259, filesize=30.4 K 2024-11-18T20:43:51,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/ca8523efac8d491abf58a161efe795e6 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/ca8523efac8d491abf58a161efe795e6 2024-11-18T20:43:52,000 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/ca8523efac8d491abf58a161efe795e6, entries=150, sequenceid=259, filesize=11.9 K 2024-11-18T20:43:52,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/81e3729f695f48d680d7539b17a0e5aa as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/81e3729f695f48d680d7539b17a0e5aa 2024-11-18T20:43:52,005 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/81e3729f695f48d680d7539b17a0e5aa, entries=150, sequenceid=259, filesize=11.9 K 2024-11-18T20:43:52,006 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for db394ce1aa4c72690b92156fd4dc5d2d in 1656ms, sequenceid=259, compaction requested=false 2024-11-18T20:43:52,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:52,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:52,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-18T20:43:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-18T20:43:52,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-18T20:43:52,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5760 sec 2024-11-18T20:43:52,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.5810 sec 2024-11-18T20:43:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:52,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T20:43:52,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:43:52,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:52,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:43:52,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:52,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:43:52,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:52,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118da6d8a460dcd499290405d25de89de45_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_0/A:col10/1731962630749/Put/seqid=0 2024-11-18T20:43:52,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742430_1606 (size=14994) 2024-11-18T20:43:52,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962692992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:52,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962692993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:53,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:53,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:53,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:53,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:53,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962693107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962693107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:53,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:53,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962693317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:53,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:53,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:53,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:53,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962693318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:53,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:53,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:53,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:53,358 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:53,361 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118da6d8a460dcd499290405d25de89de45_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118da6d8a460dcd499290405d25de89de45_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:53,362 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/47644de90c1d46729c5e389dfdf2be79, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:53,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/47644de90c1d46729c5e389dfdf2be79 is 175, key is test_row_0/A:col10/1731962630749/Put/seqid=0 2024-11-18T20:43:53,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742431_1607 (size=39949) 2024-11-18T20:43:53,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:43:53,598 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:53,598 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:43:53,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-18T20:43:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:43:53,600 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:43:53,600 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:43:53,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:43:53,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:53,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962693627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:53,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:53,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:53,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962693627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:53,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:53,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:53,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:53,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:53,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:53,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:43:53,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-18T20:43:53,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:53,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:53,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:53,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:53,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:53,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:53,766 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/47644de90c1d46729c5e389dfdf2be79 2024-11-18T20:43:53,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/6059769d5f6940259cb3cc1ac6b0e67a is 50, key is test_row_0/B:col10/1731962630749/Put/seqid=0 2024-11-18T20:43:53,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742432_1608 (size=12301) 2024-11-18T20:43:53,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-18T20:43:53,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:53,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:53,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:53,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:53,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:53,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:43:54,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-18T20:43:54,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:54,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:54,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:54,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:54,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:43:54,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962694137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:54,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:54,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962694137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:54,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/6059769d5f6940259cb3cc1ac6b0e67a 2024-11-18T20:43:54,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/c75b6cf8091e4a3ba20edfa50f687f55 is 50, key is test_row_0/C:col10/1731962630749/Put/seqid=0 2024-11-18T20:43:54,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742433_1609 (size=12301) 2024-11-18T20:43:54,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/c75b6cf8091e4a3ba20edfa50f687f55 2024-11-18T20:43:54,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/47644de90c1d46729c5e389dfdf2be79 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/47644de90c1d46729c5e389dfdf2be79 2024-11-18T20:43:54,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/47644de90c1d46729c5e389dfdf2be79, entries=200, sequenceid=274, filesize=39.0 K 2024-11-18T20:43:54,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/6059769d5f6940259cb3cc1ac6b0e67a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/6059769d5f6940259cb3cc1ac6b0e67a 2024-11-18T20:43:54,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/6059769d5f6940259cb3cc1ac6b0e67a, entries=150, sequenceid=274, filesize=12.0 K 2024-11-18T20:43:54,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/c75b6cf8091e4a3ba20edfa50f687f55 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c75b6cf8091e4a3ba20edfa50f687f55 2024-11-18T20:43:54,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c75b6cf8091e4a3ba20edfa50f687f55, entries=150, sequenceid=274, filesize=12.0 K 2024-11-18T20:43:54,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for db394ce1aa4c72690b92156fd4dc5d2d in 1274ms, sequenceid=274, compaction requested=true 2024-11-18T20:43:54,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:54,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:43:54,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:54,202 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:54,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:43:54,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:54,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:43:54,202 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:54,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:54,203 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:54,203 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:54,203 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/B is initiating minor compaction (all files) 2024-11-18T20:43:54,203 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/A is initiating minor compaction (all files) 2024-11-18T20:43:54,203 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/B in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:54,203 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/A in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:54,203 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/1b655a8adee24a579eaccd4abb9aba84, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/ca8523efac8d491abf58a161efe795e6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/6059769d5f6940259cb3cc1ac6b0e67a] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=36.4 K 2024-11-18T20:43:54,203 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6fabfe044cbf4ccc806e69fc1540e0e1, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/da018d8fd3d24466934374ed3811b160, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/47644de90c1d46729c5e389dfdf2be79] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=100.4 K 2024-11-18T20:43:54,203 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:54,203 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6fabfe044cbf4ccc806e69fc1540e0e1, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/da018d8fd3d24466934374ed3811b160, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/47644de90c1d46729c5e389dfdf2be79] 2024-11-18T20:43:54,203 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b655a8adee24a579eaccd4abb9aba84, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731962627886 2024-11-18T20:43:54,204 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fabfe044cbf4ccc806e69fc1540e0e1, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731962627886 2024-11-18T20:43:54,204 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting ca8523efac8d491abf58a161efe795e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731962628569 2024-11-18T20:43:54,204 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting da018d8fd3d24466934374ed3811b160, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731962628569 2024-11-18T20:43:54,204 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 6059769d5f6940259cb3cc1ac6b0e67a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731962630749 2024-11-18T20:43:54,204 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47644de90c1d46729c5e389dfdf2be79, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731962630749 2024-11-18T20:43:54,209 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:54,210 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#B#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:54,211 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/d2cd4778df0048dabc85bbaeb45cb516 is 50, key is test_row_0/B:col10/1731962630749/Put/seqid=0 2024-11-18T20:43:54,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-18T20:43:54,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:54,212 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T20:43:54,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:43:54,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:54,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:43:54,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:54,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:43:54,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:54,217 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111857c1fafbf6ac4411866885175e53e198_db394ce1aa4c72690b92156fd4dc5d2d store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:54,220 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111857c1fafbf6ac4411866885175e53e198_db394ce1aa4c72690b92156fd4dc5d2d, store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:54,220 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111857c1fafbf6ac4411866885175e53e198_db394ce1aa4c72690b92156fd4dc5d2d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:43:54,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742434_1610 (size=13017) 2024-11-18T20:43:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f267428bbbc14ca2842bbf6e06d0684c_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_0/A:col10/1731962632984/Put/seqid=0 2024-11-18T20:43:54,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742435_1611 (size=4469) 2024-11-18T20:43:54,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742436_1612 (size=12454) 2024-11-18T20:43:54,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:54,266 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f267428bbbc14ca2842bbf6e06d0684c_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f267428bbbc14ca2842bbf6e06d0684c_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:54,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/62cea0e0c1fb4423a5c24e1244cd87ef, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:54,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/62cea0e0c1fb4423a5c24e1244cd87ef is 175, key is test_row_0/A:col10/1731962632984/Put/seqid=0 2024-11-18T20:43:54,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742437_1613 (size=31255) 2024-11-18T20:43:54,634 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/d2cd4778df0048dabc85bbaeb45cb516 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/d2cd4778df0048dabc85bbaeb45cb516 2024-11-18T20:43:54,639 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/B of db394ce1aa4c72690b92156fd4dc5d2d into d2cd4778df0048dabc85bbaeb45cb516(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:54,639 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:54,639 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/B, priority=13, startTime=1731962634202; duration=0sec 2024-11-18T20:43:54,639 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:54,639 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:B 2024-11-18T20:43:54,639 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:43:54,640 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:43:54,640 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/C is initiating minor compaction (all files) 2024-11-18T20:43:54,640 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/C in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:54,640 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/d1c9a213e50846be8dae30aa027ee525, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/81e3729f695f48d680d7539b17a0e5aa, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c75b6cf8091e4a3ba20edfa50f687f55] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=36.4 K 2024-11-18T20:43:54,640 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting d1c9a213e50846be8dae30aa027ee525, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731962627886 2024-11-18T20:43:54,641 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 81e3729f695f48d680d7539b17a0e5aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731962628569 2024-11-18T20:43:54,641 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting c75b6cf8091e4a3ba20edfa50f687f55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731962630749 2024-11-18T20:43:54,646 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#C#compaction#518 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:54,646 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/569cd4ab6cb5485ca4a4716e41faaa5e is 50, key is test_row_0/C:col10/1731962630749/Put/seqid=0 2024-11-18T20:43:54,647 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#A#compaction#516 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:54,647 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/6034c8a44fd44402b78e832676dab8fb is 175, key is test_row_0/A:col10/1731962630749/Put/seqid=0 2024-11-18T20:43:54,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742438_1614 (size=13017) 2024-11-18T20:43:54,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742439_1615 (size=31971) 2024-11-18T20:43:54,655 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/6034c8a44fd44402b78e832676dab8fb as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6034c8a44fd44402b78e832676dab8fb 2024-11-18T20:43:54,660 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/A of db394ce1aa4c72690b92156fd4dc5d2d into 6034c8a44fd44402b78e832676dab8fb(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:54,660 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:54,660 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/A, priority=13, startTime=1731962634202; duration=0sec 2024-11-18T20:43:54,660 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:54,660 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:A 2024-11-18T20:43:54,676 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/62cea0e0c1fb4423a5c24e1244cd87ef 2024-11-18T20:43:54,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/79074859c69043faa9b2396682c9ef2c is 50, key is test_row_0/B:col10/1731962632984/Put/seqid=0 2024-11-18T20:43:54,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742440_1616 (size=12301) 2024-11-18T20:43:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:43:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:54,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:54,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35952 deadline: 1731962694889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:54,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35920 deadline: 1731962694987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:54,991 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in db394ce1aa4c72690b92156fd4dc5d2d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:54,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:54,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:54,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:55,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35952 deadline: 1731962694999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:55,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731962695018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:55,020 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_2' in db394ce1aa4c72690b92156fd4dc5d2d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:55,053 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/569cd4ab6cb5485ca4a4716e41faaa5e as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/569cd4ab6cb5485ca4a4716e41faaa5e 2024-11-18T20:43:55,058 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/C of db394ce1aa4c72690b92156fd4dc5d2d into 569cd4ab6cb5485ca4a4716e41faaa5e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:55,058 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:55,058 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/C, priority=13, startTime=1731962634202; duration=0sec 2024-11-18T20:43:55,058 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:55,058 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:C 2024-11-18T20:43:55,099 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/79074859c69043faa9b2396682c9ef2c 2024-11-18T20:43:55,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/c3d78c5fdf7e49c6bfa0067e986a90d5 is 50, key is test_row_0/C:col10/1731962632984/Put/seqid=0 2024-11-18T20:43:55,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742441_1617 (size=12301) 2024-11-18T20:43:55,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35990 deadline: 1731962695147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:55,151 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_1' in db394ce1aa4c72690b92156fd4dc5d2d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:55,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36010 deadline: 1731962695157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:55,160 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,45393,1731962440107 for 'test_row_0' in db394ce1aa4c72690b92156fd4dc5d2d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:43:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:55,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35952 deadline: 1731962695207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:55,510 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/c3d78c5fdf7e49c6bfa0067e986a90d5 2024-11-18T20:43:55,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/62cea0e0c1fb4423a5c24e1244cd87ef as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/62cea0e0c1fb4423a5c24e1244cd87ef 2024-11-18T20:43:55,519 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/62cea0e0c1fb4423a5c24e1244cd87ef, entries=150, sequenceid=293, filesize=30.5 K 2024-11-18T20:43:55,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35952 deadline: 1731962695517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:55,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:55,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:55,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:55,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/79074859c69043faa9b2396682c9ef2c as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/79074859c69043faa9b2396682c9ef2c 2024-11-18T20:43:55,523 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/79074859c69043faa9b2396682c9ef2c, entries=150, sequenceid=293, filesize=12.0 K 2024-11-18T20:43:55,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/c3d78c5fdf7e49c6bfa0067e986a90d5 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c3d78c5fdf7e49c6bfa0067e986a90d5 2024-11-18T20:43:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,527 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c3d78c5fdf7e49c6bfa0067e986a90d5, entries=150, sequenceid=293, filesize=12.0 K 2024-11-18T20:43:55,527 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for db394ce1aa4c72690b92156fd4dc5d2d in 1315ms, sequenceid=293, compaction requested=false 2024-11-18T20:43:55,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:55,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:55,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-18T20:43:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-18T20:43:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,531 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-18T20:43:55,531 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9290 sec 2024-11-18T20:43:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.9330 sec 2024-11-18T20:43:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:43:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,747 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,748 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:43:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-18T20:43:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T20:43:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,751 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:43:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,752 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:43:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:43:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T20:43:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-18T20:43:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,903 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T20:43:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:43:55,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:55,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:43:55,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:55,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:43:55,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118862a9a5348ce4745a9015ebb5f5eb9a5_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_0/A:col10/1731962634876/Put/seqid=0 2024-11-18T20:43:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742442_1618 (size=9914) 2024-11-18T20:43:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,934 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118862a9a5348ce4745a9015ebb5f5eb9a5_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118862a9a5348ce4745a9015ebb5f5eb9a5_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/0e108eae66294a6fb9e01a338b671e3a, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:55,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/0e108eae66294a6fb9e01a338b671e3a is 175, key is test_row_0/A:col10/1731962634876/Put/seqid=0 2024-11-18T20:43:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742443_1619 (size=22561) 2024-11-18T20:43:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,944 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/0e108eae66294a6fb9e01a338b671e3a 2024-11-18T20:43:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/9b55dadb5c584a4089847bc22d7fa1f2 is 50, key is test_row_0/B:col10/1731962634876/Put/seqid=0 2024-11-18T20:43:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742444_1620 (size=9857) 2024-11-18T20:43:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:56,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. as already flushing 2024-11-18T20:43:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45393 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T20:43:56,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35952 deadline: 1731962696111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:56,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:56,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:56,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:56,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35952 deadline: 1731962696217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:56,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:56,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:56,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:56,353 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/9b55dadb5c584a4089847bc22d7fa1f2 2024-11-18T20:43:56,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/5a6431ef1cee4383aaf3bee0f313d311 is 50, key is test_row_0/C:col10/1731962634876/Put/seqid=0 2024-11-18T20:43:56,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742445_1621 (size=9857) 2024-11-18T20:43:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T20:43:56,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35952 deadline: 1731962696427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:56,527 INFO [Thread-2371 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2371. 2024-11-18T20:43:56,527 DEBUG [Thread-2371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:43:56,527 DEBUG [Thread-2371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:56,527 DEBUG [Thread-2371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:56,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:56,527 INFO [Thread-2373 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2373. 2024-11-18T20:43:56,527 DEBUG [Thread-2373 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:43:56,527 DEBUG [Thread-2373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:56,527 DEBUG [Thread-2373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:56,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:56,529 INFO [Thread-2375 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2375. 2024-11-18T20:43:56,529 DEBUG [Thread-2375 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:43:56,529 DEBUG [Thread-2375 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:56,529 DEBUG [Thread-2375 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:56,529 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:56,530 INFO [Thread-2367 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2367. 2024-11-18T20:43:56,530 DEBUG [Thread-2367 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:43:56,530 DEBUG [Thread-2367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:56,530 DEBUG [Thread-2367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:56,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:56,530 INFO [Thread-2369 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2369. 2024-11-18T20:43:56,530 DEBUG [Thread-2369 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:43:56,530 DEBUG [Thread-2369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:56,530 DEBUG [Thread-2369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:56,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:56,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:43:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35952 deadline: 1731962696737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 2024-11-18T20:43:56,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 , the old value is region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:56,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:43:56,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., hostname=c0a89b2656d4,45393,1731962440107, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:43:56,764 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/5a6431ef1cee4383aaf3bee0f313d311 2024-11-18T20:43:56,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/0e108eae66294a6fb9e01a338b671e3a as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/0e108eae66294a6fb9e01a338b671e3a 2024-11-18T20:43:56,772 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/0e108eae66294a6fb9e01a338b671e3a, entries=100, sequenceid=309, filesize=22.0 K 2024-11-18T20:43:56,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/9b55dadb5c584a4089847bc22d7fa1f2 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/9b55dadb5c584a4089847bc22d7fa1f2 2024-11-18T20:43:56,776 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/9b55dadb5c584a4089847bc22d7fa1f2, entries=100, sequenceid=309, filesize=9.6 K 2024-11-18T20:43:56,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/5a6431ef1cee4383aaf3bee0f313d311 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/5a6431ef1cee4383aaf3bee0f313d311 2024-11-18T20:43:56,781 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/5a6431ef1cee4383aaf3bee0f313d311, entries=100, sequenceid=309, filesize=9.6 K 2024-11-18T20:43:56,782 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for db394ce1aa4c72690b92156fd4dc5d2d in 879ms, sequenceid=309, compaction requested=true 2024-11-18T20:43:56,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:56,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:56,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-18T20:43:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-18T20:43:56,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-18T20:43:56,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0320 sec 2024-11-18T20:43:56,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.0380 sec 2024-11-18T20:43:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T20:43:56,888 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:43:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45393 {}] regionserver.HRegion(8855): Flush requested on db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:57,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T20:43:57,169 INFO [Thread-2358 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2358. 2024-11-18T20:43:57,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:43:57,169 DEBUG [Thread-2358 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:43:57,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:57,169 DEBUG [Thread-2358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:57,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:43:57,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:57,169 DEBUG [Thread-2358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:57,169 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:57,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:43:57,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:43:57,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bc1530d18c9847ef9d52d5dd50f88092_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_0/A:col10/1731962637168/Put/seqid=0 2024-11-18T20:43:57,179 INFO [Thread-2364 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2364. 2024-11-18T20:43:57,179 DEBUG [Thread-2364 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:43:57,179 DEBUG [Thread-2364 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:57,179 DEBUG [Thread-2364 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:57,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:57,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742446_1622 (size=12454) 2024-11-18T20:43:57,249 INFO [Thread-2362 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2362. 2024-11-18T20:43:57,249 DEBUG [Thread-2362 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:43:57,249 DEBUG [Thread-2362 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:57,250 DEBUG [Thread-2362 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:43:57,250 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:43:57,581 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:43:57,584 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bc1530d18c9847ef9d52d5dd50f88092_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bc1530d18c9847ef9d52d5dd50f88092_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:43:57,585 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/b5a0277808a84b478d40d149bd680578, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:57,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/b5a0277808a84b478d40d149bd680578 is 175, key is test_row_0/A:col10/1731962637168/Put/seqid=0 2024-11-18T20:43:57,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742447_1623 (size=31255) 2024-11-18T20:43:57,589 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/b5a0277808a84b478d40d149bd680578 2024-11-18T20:43:57,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/4344a65e18f44a1a8192f38a45b31771 is 50, key is test_row_0/B:col10/1731962637168/Put/seqid=0 2024-11-18T20:43:57,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742448_1624 (size=12301) 2024-11-18T20:43:58,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/4344a65e18f44a1a8192f38a45b31771 2024-11-18T20:43:58,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/f246667982954f02807f5484754b799d is 50, key is test_row_0/C:col10/1731962637168/Put/seqid=0 2024-11-18T20:43:58,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742449_1625 (size=12301) 2024-11-18T20:43:58,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/f246667982954f02807f5484754b799d 2024-11-18T20:43:58,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/b5a0277808a84b478d40d149bd680578 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/b5a0277808a84b478d40d149bd680578 2024-11-18T20:43:58,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/b5a0277808a84b478d40d149bd680578, entries=150, sequenceid=329, filesize=30.5 K 2024-11-18T20:43:58,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/4344a65e18f44a1a8192f38a45b31771 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/4344a65e18f44a1a8192f38a45b31771 2024-11-18T20:43:58,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/4344a65e18f44a1a8192f38a45b31771, entries=150, sequenceid=329, filesize=12.0 K 2024-11-18T20:43:58,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/f246667982954f02807f5484754b799d as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/f246667982954f02807f5484754b799d 2024-11-18T20:43:58,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/f246667982954f02807f5484754b799d, entries=150, sequenceid=329, filesize=12.0 K 2024-11-18T20:43:58,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=13.42 KB/13740 for db394ce1aa4c72690b92156fd4dc5d2d in 1273ms, sequenceid=329, compaction requested=true 2024-11-18T20:43:58,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:58,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:43:58,442 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:43:58,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:58,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:43:58,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:58,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store db394ce1aa4c72690b92156fd4dc5d2d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:43:58,442 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:43:58,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:58,443 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 117042 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:43:58,443 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/A is initiating minor compaction (all files) 2024-11-18T20:43:58,443 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/A in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:58,443 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:43:58,443 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/B is initiating minor compaction (all files) 2024-11-18T20:43:58,443 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6034c8a44fd44402b78e832676dab8fb, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/62cea0e0c1fb4423a5c24e1244cd87ef, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/0e108eae66294a6fb9e01a338b671e3a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/b5a0277808a84b478d40d149bd680578] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=114.3 K 2024-11-18T20:43:58,443 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:58,443 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/B in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:58,443 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. files: [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6034c8a44fd44402b78e832676dab8fb, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/62cea0e0c1fb4423a5c24e1244cd87ef, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/0e108eae66294a6fb9e01a338b671e3a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/b5a0277808a84b478d40d149bd680578] 2024-11-18T20:43:58,444 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/d2cd4778df0048dabc85bbaeb45cb516, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/79074859c69043faa9b2396682c9ef2c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/9b55dadb5c584a4089847bc22d7fa1f2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/4344a65e18f44a1a8192f38a45b31771] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=46.4 K 2024-11-18T20:43:58,444 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6034c8a44fd44402b78e832676dab8fb, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731962630749 2024-11-18T20:43:58,444 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting d2cd4778df0048dabc85bbaeb45cb516, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731962630749 2024-11-18T20:43:58,444 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62cea0e0c1fb4423a5c24e1244cd87ef, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731962632984 2024-11-18T20:43:58,445 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e108eae66294a6fb9e01a338b671e3a, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731962634876 2024-11-18T20:43:58,445 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 79074859c69043faa9b2396682c9ef2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731962632984 2024-11-18T20:43:58,445 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5a0277808a84b478d40d149bd680578, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731962636105 2024-11-18T20:43:58,445 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b55dadb5c584a4089847bc22d7fa1f2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731962634876 2024-11-18T20:43:58,445 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 4344a65e18f44a1a8192f38a45b31771, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731962636105 2024-11-18T20:43:58,451 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:58,453 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118c2de8ff26e38411c805efd189bf0580d_db394ce1aa4c72690b92156fd4dc5d2d store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:58,453 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#B#compaction#527 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:58,453 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/ac5c2c57ffe84bb2ab6b68cb5d4c93a7 is 50, key is test_row_0/B:col10/1731962637168/Put/seqid=0 2024-11-18T20:43:58,455 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118c2de8ff26e38411c805efd189bf0580d_db394ce1aa4c72690b92156fd4dc5d2d, store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:58,455 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c2de8ff26e38411c805efd189bf0580d_db394ce1aa4c72690b92156fd4dc5d2d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:43:58,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742450_1626 (size=13153) 2024-11-18T20:43:58,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742451_1627 (size=4469) 2024-11-18T20:43:58,862 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/ac5c2c57ffe84bb2ab6b68cb5d4c93a7 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/ac5c2c57ffe84bb2ab6b68cb5d4c93a7 2024-11-18T20:43:58,863 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#A#compaction#528 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:58,864 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/9efc740412ce41cc8dbc76e1ba7e6aa8 is 175, key is test_row_0/A:col10/1731962637168/Put/seqid=0 2024-11-18T20:43:58,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742452_1628 (size=32107) 2024-11-18T20:43:58,868 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/B of db394ce1aa4c72690b92156fd4dc5d2d into ac5c2c57ffe84bb2ab6b68cb5d4c93a7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:58,868 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:58,868 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/B, priority=12, startTime=1731962638442; duration=0sec 2024-11-18T20:43:58,868 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:43:58,868 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:B 2024-11-18T20:43:58,869 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:43:58,870 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:43:58,870 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1541): db394ce1aa4c72690b92156fd4dc5d2d/C is initiating minor compaction (all files) 2024-11-18T20:43:58,870 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of db394ce1aa4c72690b92156fd4dc5d2d/C in TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:43:58,870 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/569cd4ab6cb5485ca4a4716e41faaa5e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c3d78c5fdf7e49c6bfa0067e986a90d5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/5a6431ef1cee4383aaf3bee0f313d311, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/f246667982954f02807f5484754b799d] into tmpdir=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp, totalSize=46.4 K 2024-11-18T20:43:58,871 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 569cd4ab6cb5485ca4a4716e41faaa5e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731962630749 2024-11-18T20:43:58,871 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting c3d78c5fdf7e49c6bfa0067e986a90d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731962632984 2024-11-18T20:43:58,871 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a6431ef1cee4383aaf3bee0f313d311, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731962634876 2024-11-18T20:43:58,872 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] compactions.Compactor(225): Compacting f246667982954f02807f5484754b799d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731962636105 2024-11-18T20:43:58,880 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): db394ce1aa4c72690b92156fd4dc5d2d#C#compaction#529 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:43:58,880 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/d46fa0a2aa404a28b4d5ad6a3f51a697 is 50, key is test_row_0/C:col10/1731962637168/Put/seqid=0 2024-11-18T20:43:58,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742453_1629 (size=13153) 2024-11-18T20:43:59,272 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/9efc740412ce41cc8dbc76e1ba7e6aa8 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/9efc740412ce41cc8dbc76e1ba7e6aa8 2024-11-18T20:43:59,277 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/A of db394ce1aa4c72690b92156fd4dc5d2d into 9efc740412ce41cc8dbc76e1ba7e6aa8(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:59,277 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:59,277 INFO [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/A, priority=12, startTime=1731962638442; duration=0sec 2024-11-18T20:43:59,277 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:59,277 DEBUG [RS:0;c0a89b2656d4:45393-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:A 2024-11-18T20:43:59,289 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/d46fa0a2aa404a28b4d5ad6a3f51a697 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/d46fa0a2aa404a28b4d5ad6a3f51a697 2024-11-18T20:43:59,294 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in db394ce1aa4c72690b92156fd4dc5d2d/C of db394ce1aa4c72690b92156fd4dc5d2d into d46fa0a2aa404a28b4d5ad6a3f51a697(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:43:59,294 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for db394ce1aa4c72690b92156fd4dc5d2d: 2024-11-18T20:43:59,294 INFO [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., storeName=db394ce1aa4c72690b92156fd4dc5d2d/C, priority=12, startTime=1731962638442; duration=0sec 2024-11-18T20:43:59,294 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:43:59,294 DEBUG [RS:0;c0a89b2656d4:45393-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: db394ce1aa4c72690b92156fd4dc5d2d:C 2024-11-18T20:44:05,019 INFO [Thread-2356 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2356. 2024-11-18T20:44:05,020 DEBUG [Thread-2356 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:44:05,020 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:44:05,020 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:44:05,020 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:44:05,129 INFO [Thread-2360 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2360. 2024-11-18T20:44:05,129 DEBUG [Thread-2360 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:44:05,129 DEBUG [Thread-2360 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:44:05,129 DEBUG [Thread-2360 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:44:05,129 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:44:05,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T20:44:05,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-18T20:44:05,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-18T20:44:05,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-18T20:44:05,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-18T20:44:05,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-18T20:44:05,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T20:44:05,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3592 2024-11-18T20:44:05,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3205 2024-11-18T20:44:05,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3161 2024-11-18T20:44:05,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3600 2024-11-18T20:44:05,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3211 2024-11-18T20:44:05,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T20:44:05,130 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:44:05,130 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:44:05,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:44:05,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:44:05,130 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:44:05,131 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T20:44:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T20:44:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T20:44:05,134 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962645134"}]},"ts":"1731962645134"} 2024-11-18T20:44:05,135 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T20:44:05,135 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T20:44:05,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:44:05,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=db394ce1aa4c72690b92156fd4dc5d2d, UNASSIGN}] 2024-11-18T20:44:05,138 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=db394ce1aa4c72690b92156fd4dc5d2d, UNASSIGN 2024-11-18T20:44:05,139 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=db394ce1aa4c72690b92156fd4dc5d2d, regionState=CLOSING, regionLocation=c0a89b2656d4,45393,1731962440107 2024-11-18T20:44:05,139 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46825 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=c0a89b2656d4,45393,1731962440107, table=TestAcidGuarantees, region=db394ce1aa4c72690b92156fd4dc5d2d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-18T20:44:05,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=db394ce1aa4c72690b92156fd4dc5d2d, UNASSIGN because future has completed 2024-11-18T20:44:05,141 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:44:05,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107}] 2024-11-18T20:44:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T20:44:05,295 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:05,295 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:44:05,295 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing db394ce1aa4c72690b92156fd4dc5d2d, disabling compactions & flushes 2024-11-18T20:44:05,295 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:44:05,295 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:44:05,295 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. after waiting 0 ms 2024-11-18T20:44:05,295 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:44:05,295 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing db394ce1aa4c72690b92156fd4dc5d2d 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-18T20:44:05,295 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=A 2024-11-18T20:44:05,295 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:44:05,295 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=B 2024-11-18T20:44:05,295 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:44:05,295 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK db394ce1aa4c72690b92156fd4dc5d2d, store=C 2024-11-18T20:44:05,295 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T20:44:05,300 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111809bb56aafb094fceb70506cd205de139_db394ce1aa4c72690b92156fd4dc5d2d is 50, key is test_row_1/A:col10/1731962645018/Put/seqid=0 2024-11-18T20:44:05,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742454_1630 (size=9914) 2024-11-18T20:44:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T20:44:05,703 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:44:05,706 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111809bb56aafb094fceb70506cd205de139_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111809bb56aafb094fceb70506cd205de139_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:05,707 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/6456eb2c85944c6dabf124a53031e1f9, store: [table=TestAcidGuarantees family=A region=db394ce1aa4c72690b92156fd4dc5d2d] 2024-11-18T20:44:05,708 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/6456eb2c85944c6dabf124a53031e1f9 is 175, key is test_row_1/A:col10/1731962645018/Put/seqid=0 2024-11-18T20:44:05,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742455_1631 (size=22561) 2024-11-18T20:44:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T20:44:06,112 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/6456eb2c85944c6dabf124a53031e1f9 2024-11-18T20:44:06,117 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/36a11b6c6b554e0b9835e1089a0a9f4f is 50, key is test_row_1/B:col10/1731962645018/Put/seqid=0 2024-11-18T20:44:06,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742456_1632 (size=9857) 2024-11-18T20:44:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T20:44:06,521 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/36a11b6c6b554e0b9835e1089a0a9f4f 2024-11-18T20:44:06,526 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/11ac2c3def884f689590e04afd78395e is 50, key is test_row_1/C:col10/1731962645018/Put/seqid=0 2024-11-18T20:44:06,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742457_1633 (size=9857) 2024-11-18T20:44:06,930 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/11ac2c3def884f689590e04afd78395e 2024-11-18T20:44:06,934 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/A/6456eb2c85944c6dabf124a53031e1f9 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6456eb2c85944c6dabf124a53031e1f9 2024-11-18T20:44:06,939 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6456eb2c85944c6dabf124a53031e1f9, entries=100, sequenceid=339, filesize=22.0 K 2024-11-18T20:44:06,941 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/B/36a11b6c6b554e0b9835e1089a0a9f4f as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/36a11b6c6b554e0b9835e1089a0a9f4f 2024-11-18T20:44:06,946 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/36a11b6c6b554e0b9835e1089a0a9f4f, entries=100, sequenceid=339, filesize=9.6 K 2024-11-18T20:44:06,947 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/.tmp/C/11ac2c3def884f689590e04afd78395e as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/11ac2c3def884f689590e04afd78395e 2024-11-18T20:44:06,952 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/11ac2c3def884f689590e04afd78395e, entries=100, sequenceid=339, filesize=9.6 K 2024-11-18T20:44:06,953 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for db394ce1aa4c72690b92156fd4dc5d2d in 1658ms, sequenceid=339, compaction requested=false 2024-11-18T20:44:06,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/49bdc754670b457cb0191f6e33ff71c8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/bb7ad99e490e47c6acefab19ed3d5c52, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c4690563cd7347bba28bb0b0ac36658f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c89be9009c8b4503bb5208f138f80db8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/d1351b88e6a94c38b7d1c37689025037, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/3d85e2fc001d49da9af2f5020557b597, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/422facf368164c01a20fc89623e89d91, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/439df09e7bfc4f068ac1a0e09e88e437, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6966e2f518fa4667b10350c8e5cc1ab7, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c77fedc983714974a8622cf14c80013a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/099c46959e084b158fec4c4b19f0b89b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/3eeee260dfd84c02b0e6947ed1d5191f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/137389d95d424f618f2620bf9aa7bce8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/8ad2b2276fb54cee9a4ca266624ca338, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/ee15a28f0e954f30b79f66bbcfaeaaba, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/4d334d05ce1741c59693ec1633ba2692, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/91291f8452844b2996af2733457189cc, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/8c9cd6649ca041c99d3bfb7986c077e0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6fabfe044cbf4ccc806e69fc1540e0e1, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/da018d8fd3d24466934374ed3811b160, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/47644de90c1d46729c5e389dfdf2be79, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6034c8a44fd44402b78e832676dab8fb, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/62cea0e0c1fb4423a5c24e1244cd87ef, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/0e108eae66294a6fb9e01a338b671e3a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/b5a0277808a84b478d40d149bd680578] to archive 2024-11-18T20:44:06,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:44:06,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/49bdc754670b457cb0191f6e33ff71c8 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/49bdc754670b457cb0191f6e33ff71c8 2024-11-18T20:44:06,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/bb7ad99e490e47c6acefab19ed3d5c52 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/bb7ad99e490e47c6acefab19ed3d5c52 2024-11-18T20:44:06,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c4690563cd7347bba28bb0b0ac36658f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c4690563cd7347bba28bb0b0ac36658f 2024-11-18T20:44:06,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c89be9009c8b4503bb5208f138f80db8 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c89be9009c8b4503bb5208f138f80db8 2024-11-18T20:44:06,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/d1351b88e6a94c38b7d1c37689025037 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/d1351b88e6a94c38b7d1c37689025037 2024-11-18T20:44:06,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/3d85e2fc001d49da9af2f5020557b597 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/3d85e2fc001d49da9af2f5020557b597 2024-11-18T20:44:06,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/422facf368164c01a20fc89623e89d91 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/422facf368164c01a20fc89623e89d91 2024-11-18T20:44:06,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/439df09e7bfc4f068ac1a0e09e88e437 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/439df09e7bfc4f068ac1a0e09e88e437 2024-11-18T20:44:06,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6966e2f518fa4667b10350c8e5cc1ab7 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6966e2f518fa4667b10350c8e5cc1ab7 2024-11-18T20:44:06,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c77fedc983714974a8622cf14c80013a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/c77fedc983714974a8622cf14c80013a 2024-11-18T20:44:06,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/099c46959e084b158fec4c4b19f0b89b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/099c46959e084b158fec4c4b19f0b89b 2024-11-18T20:44:06,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/3eeee260dfd84c02b0e6947ed1d5191f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/3eeee260dfd84c02b0e6947ed1d5191f 2024-11-18T20:44:06,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/137389d95d424f618f2620bf9aa7bce8 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/137389d95d424f618f2620bf9aa7bce8 2024-11-18T20:44:06,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/8ad2b2276fb54cee9a4ca266624ca338 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/8ad2b2276fb54cee9a4ca266624ca338 2024-11-18T20:44:06,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/ee15a28f0e954f30b79f66bbcfaeaaba to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/ee15a28f0e954f30b79f66bbcfaeaaba 2024-11-18T20:44:06,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/4d334d05ce1741c59693ec1633ba2692 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/4d334d05ce1741c59693ec1633ba2692 2024-11-18T20:44:06,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/91291f8452844b2996af2733457189cc to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/91291f8452844b2996af2733457189cc 2024-11-18T20:44:06,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/8c9cd6649ca041c99d3bfb7986c077e0 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/8c9cd6649ca041c99d3bfb7986c077e0 2024-11-18T20:44:06,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6fabfe044cbf4ccc806e69fc1540e0e1 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6fabfe044cbf4ccc806e69fc1540e0e1 2024-11-18T20:44:06,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/da018d8fd3d24466934374ed3811b160 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/da018d8fd3d24466934374ed3811b160 2024-11-18T20:44:06,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/47644de90c1d46729c5e389dfdf2be79 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/47644de90c1d46729c5e389dfdf2be79 2024-11-18T20:44:06,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6034c8a44fd44402b78e832676dab8fb to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6034c8a44fd44402b78e832676dab8fb 2024-11-18T20:44:06,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/62cea0e0c1fb4423a5c24e1244cd87ef to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/62cea0e0c1fb4423a5c24e1244cd87ef 2024-11-18T20:44:06,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/0e108eae66294a6fb9e01a338b671e3a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/0e108eae66294a6fb9e01a338b671e3a 2024-11-18T20:44:06,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/b5a0277808a84b478d40d149bd680578 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/b5a0277808a84b478d40d149bd680578 2024-11-18T20:44:06,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/c38efdabed35477bb5da875d1cedc997, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/8d5c88e43da94060bd5a147e497ea839, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/b367ea65bb8844afb7f5ffd5f5444a60, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/870af5227e58480a8de1a7461a428e8b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/32b2ac82079245eda3b0bf65e51c98d5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/3e8ec9ec41564d8eae67781bc506a96b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/e1ea1ad8cacb4727b09c306ce1727019, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/c15212ece9744c56aa60022363c7c0e5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/dccf41922d8f4e008d8aec59e7e2acce, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/0f5132ae21164915865fe8cdc1f84913, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/ea6fbad15ed74d7dac2422f04f65d05d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/9f59697215374e058e1154e99d942cb4, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/e2524d8adf7a4deaaeda4806ef0a4fe4, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/3a91c2f23a714184bfa616bcb2577fcb, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/d2a6f8bfe5b54c728a933b5eb23b5641, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/b64d96ae03914cf9b07ddebd7c174ec2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/fd330a522751423f8877b10d69e8e66e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/1b655a8adee24a579eaccd4abb9aba84, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/4fd0e383f92f4789877c776e12bba4ce, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/ca8523efac8d491abf58a161efe795e6, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/d2cd4778df0048dabc85bbaeb45cb516, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/6059769d5f6940259cb3cc1ac6b0e67a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/79074859c69043faa9b2396682c9ef2c, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/9b55dadb5c584a4089847bc22d7fa1f2, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/4344a65e18f44a1a8192f38a45b31771] to archive 2024-11-18T20:44:06,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:44:06,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/c38efdabed35477bb5da875d1cedc997 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/c38efdabed35477bb5da875d1cedc997 2024-11-18T20:44:06,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/8d5c88e43da94060bd5a147e497ea839 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/8d5c88e43da94060bd5a147e497ea839 2024-11-18T20:44:06,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/b367ea65bb8844afb7f5ffd5f5444a60 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/b367ea65bb8844afb7f5ffd5f5444a60 2024-11-18T20:44:06,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/870af5227e58480a8de1a7461a428e8b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/870af5227e58480a8de1a7461a428e8b 2024-11-18T20:44:06,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/32b2ac82079245eda3b0bf65e51c98d5 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/32b2ac82079245eda3b0bf65e51c98d5 2024-11-18T20:44:06,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/3e8ec9ec41564d8eae67781bc506a96b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/3e8ec9ec41564d8eae67781bc506a96b 2024-11-18T20:44:06,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/e1ea1ad8cacb4727b09c306ce1727019 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/e1ea1ad8cacb4727b09c306ce1727019 2024-11-18T20:44:06,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/c15212ece9744c56aa60022363c7c0e5 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/c15212ece9744c56aa60022363c7c0e5 2024-11-18T20:44:06,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/dccf41922d8f4e008d8aec59e7e2acce to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/dccf41922d8f4e008d8aec59e7e2acce 2024-11-18T20:44:06,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/0f5132ae21164915865fe8cdc1f84913 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/0f5132ae21164915865fe8cdc1f84913 2024-11-18T20:44:06,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/ea6fbad15ed74d7dac2422f04f65d05d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/ea6fbad15ed74d7dac2422f04f65d05d 2024-11-18T20:44:06,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/9f59697215374e058e1154e99d942cb4 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/9f59697215374e058e1154e99d942cb4 2024-11-18T20:44:06,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/e2524d8adf7a4deaaeda4806ef0a4fe4 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/e2524d8adf7a4deaaeda4806ef0a4fe4 2024-11-18T20:44:06,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/3a91c2f23a714184bfa616bcb2577fcb to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/3a91c2f23a714184bfa616bcb2577fcb 2024-11-18T20:44:06,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/d2a6f8bfe5b54c728a933b5eb23b5641 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/d2a6f8bfe5b54c728a933b5eb23b5641 2024-11-18T20:44:06,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/b64d96ae03914cf9b07ddebd7c174ec2 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/b64d96ae03914cf9b07ddebd7c174ec2 2024-11-18T20:44:06,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/fd330a522751423f8877b10d69e8e66e to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/fd330a522751423f8877b10d69e8e66e 2024-11-18T20:44:06,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/1b655a8adee24a579eaccd4abb9aba84 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/1b655a8adee24a579eaccd4abb9aba84 2024-11-18T20:44:06,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/4fd0e383f92f4789877c776e12bba4ce to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/4fd0e383f92f4789877c776e12bba4ce 2024-11-18T20:44:06,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/ca8523efac8d491abf58a161efe795e6 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/ca8523efac8d491abf58a161efe795e6 2024-11-18T20:44:06,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/d2cd4778df0048dabc85bbaeb45cb516 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/d2cd4778df0048dabc85bbaeb45cb516 2024-11-18T20:44:06,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/6059769d5f6940259cb3cc1ac6b0e67a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/6059769d5f6940259cb3cc1ac6b0e67a 2024-11-18T20:44:07,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/79074859c69043faa9b2396682c9ef2c to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/79074859c69043faa9b2396682c9ef2c 2024-11-18T20:44:07,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/9b55dadb5c584a4089847bc22d7fa1f2 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/9b55dadb5c584a4089847bc22d7fa1f2 2024-11-18T20:44:07,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/4344a65e18f44a1a8192f38a45b31771 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/4344a65e18f44a1a8192f38a45b31771 2024-11-18T20:44:07,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/f6918feadb994f8fad680f81226f8b0f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/bb8636b6feea4b79898f17572988f2a1, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/931854dead8044948bfc250472dbe16a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/26fe6c39e88f48a6b5b6f52e117dd5db, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/ea464c81a53349c9a0670ed9649a6952, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c2b4225f3d1f4362b607b78c4bb2c110, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/62580eaea1c2400287cc849709c0128a, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/744c80bd0a3d4e199401f442e1a032b8, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/15aa6fd697f94db18092069c3a089a62, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c75e54db3bc34cbaaed82e90ed6d720f, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/ed59148c1a474781ac39cd8004931833, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/05565d2248174e4fa5de48d59f0d3c59, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/8dc1fdfa4bc34150a5d48a39849a796d, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/78508445e18046a8af40c6aede6930ca, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/3e1c762614824236a05b7b4a47cc39d0, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/4422348d1e464368a925afaf4f1a1243, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/7a9ebe89b2d54030a665818339a67582, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/d1c9a213e50846be8dae30aa027ee525, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/98649dda6d334903a2b987d8c70cb92b, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/81e3729f695f48d680d7539b17a0e5aa, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/569cd4ab6cb5485ca4a4716e41faaa5e, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c75b6cf8091e4a3ba20edfa50f687f55, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c3d78c5fdf7e49c6bfa0067e986a90d5, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/5a6431ef1cee4383aaf3bee0f313d311, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/f246667982954f02807f5484754b799d] to archive 2024-11-18T20:44:07,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:44:07,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/f6918feadb994f8fad680f81226f8b0f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/f6918feadb994f8fad680f81226f8b0f 2024-11-18T20:44:07,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/bb8636b6feea4b79898f17572988f2a1 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/bb8636b6feea4b79898f17572988f2a1 2024-11-18T20:44:07,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/931854dead8044948bfc250472dbe16a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/931854dead8044948bfc250472dbe16a 2024-11-18T20:44:07,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/26fe6c39e88f48a6b5b6f52e117dd5db to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/26fe6c39e88f48a6b5b6f52e117dd5db 2024-11-18T20:44:07,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/ea464c81a53349c9a0670ed9649a6952 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/ea464c81a53349c9a0670ed9649a6952 2024-11-18T20:44:07,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c2b4225f3d1f4362b607b78c4bb2c110 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c2b4225f3d1f4362b607b78c4bb2c110 2024-11-18T20:44:07,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/62580eaea1c2400287cc849709c0128a to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/62580eaea1c2400287cc849709c0128a 2024-11-18T20:44:07,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/744c80bd0a3d4e199401f442e1a032b8 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/744c80bd0a3d4e199401f442e1a032b8 2024-11-18T20:44:07,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/15aa6fd697f94db18092069c3a089a62 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/15aa6fd697f94db18092069c3a089a62 2024-11-18T20:44:07,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c75e54db3bc34cbaaed82e90ed6d720f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c75e54db3bc34cbaaed82e90ed6d720f 2024-11-18T20:44:07,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/ed59148c1a474781ac39cd8004931833 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/ed59148c1a474781ac39cd8004931833 2024-11-18T20:44:07,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/05565d2248174e4fa5de48d59f0d3c59 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/05565d2248174e4fa5de48d59f0d3c59 2024-11-18T20:44:07,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/8dc1fdfa4bc34150a5d48a39849a796d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/8dc1fdfa4bc34150a5d48a39849a796d 2024-11-18T20:44:07,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/78508445e18046a8af40c6aede6930ca to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/78508445e18046a8af40c6aede6930ca 2024-11-18T20:44:07,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/3e1c762614824236a05b7b4a47cc39d0 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/3e1c762614824236a05b7b4a47cc39d0 2024-11-18T20:44:07,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/4422348d1e464368a925afaf4f1a1243 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/4422348d1e464368a925afaf4f1a1243 2024-11-18T20:44:07,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/7a9ebe89b2d54030a665818339a67582 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/7a9ebe89b2d54030a665818339a67582 2024-11-18T20:44:07,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/d1c9a213e50846be8dae30aa027ee525 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/d1c9a213e50846be8dae30aa027ee525 2024-11-18T20:44:07,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/98649dda6d334903a2b987d8c70cb92b to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/98649dda6d334903a2b987d8c70cb92b 2024-11-18T20:44:07,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/81e3729f695f48d680d7539b17a0e5aa to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/81e3729f695f48d680d7539b17a0e5aa 2024-11-18T20:44:07,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/569cd4ab6cb5485ca4a4716e41faaa5e to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/569cd4ab6cb5485ca4a4716e41faaa5e 2024-11-18T20:44:07,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c75b6cf8091e4a3ba20edfa50f687f55 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c75b6cf8091e4a3ba20edfa50f687f55 2024-11-18T20:44:07,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c3d78c5fdf7e49c6bfa0067e986a90d5 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/c3d78c5fdf7e49c6bfa0067e986a90d5 2024-11-18T20:44:07,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/5a6431ef1cee4383aaf3bee0f313d311 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/5a6431ef1cee4383aaf3bee0f313d311 2024-11-18T20:44:07,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/f246667982954f02807f5484754b799d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/f246667982954f02807f5484754b799d 2024-11-18T20:44:07,029 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/recovered.edits/342.seqid, newMaxSeqId=342, maxSeqId=4 2024-11-18T20:44:07,030 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. 2024-11-18T20:44:07,030 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for db394ce1aa4c72690b92156fd4dc5d2d: Waiting for close lock at 1731962645295Running coprocessor pre-close hooks at 1731962645295Disabling compacts and flushes for region at 1731962645295Disabling writes for close at 1731962645295Obtaining lock to block concurrent updates at 1731962645295Preparing flush snapshotting stores in db394ce1aa4c72690b92156fd4dc5d2d at 1731962645295Finished memstore snapshotting TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731962645295Flushing stores of TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d. at 1731962645296 (+1 ms)Flushing db394ce1aa4c72690b92156fd4dc5d2d/A: creating writer at 1731962645296Flushing mob file db394ce1aa4c72690b92156fd4dc5d2d/A: appending metadata at 1731962645299 (+3 ms)Flushing mob file db394ce1aa4c72690b92156fd4dc5d2d/A: closing flushed file at 1731962645300 (+1 ms)Flushing db394ce1aa4c72690b92156fd4dc5d2d/A: appending metadata at 1731962645707 (+407 ms)Flushing db394ce1aa4c72690b92156fd4dc5d2d/A: closing flushed file at 1731962645707Flushing db394ce1aa4c72690b92156fd4dc5d2d/B: creating writer at 1731962646115 (+408 ms)Flushing db394ce1aa4c72690b92156fd4dc5d2d/B: appending metadata at 1731962646117 (+2 ms)Flushing db394ce1aa4c72690b92156fd4dc5d2d/B: closing flushed file at 1731962646117Flushing db394ce1aa4c72690b92156fd4dc5d2d/C: creating writer at 1731962646524 (+407 ms)Flushing db394ce1aa4c72690b92156fd4dc5d2d/C: appending metadata at 1731962646526 (+2 ms)Flushing db394ce1aa4c72690b92156fd4dc5d2d/C: closing flushed file at 1731962646526Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4971bd58: reopening flushed file at 1731962646933 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@549a7edb: reopening flushed file at 1731962646939 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72011c98: reopening flushed file at 1731962646946 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for db394ce1aa4c72690b92156fd4dc5d2d in 1658ms, sequenceid=339, compaction requested=false at 1731962646953 (+7 ms)Writing region close event to WAL at 1731962647027 (+74 ms)Running coprocessor post-close hooks at 1731962647030 (+3 ms)Closed at 1731962647030 2024-11-18T20:44:07,032 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,032 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=db394ce1aa4c72690b92156fd4dc5d2d, regionState=CLOSED 2024-11-18T20:44:07,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 because future has completed 2024-11-18T20:44:07,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-18T20:44:07,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure db394ce1aa4c72690b92156fd4dc5d2d, server=c0a89b2656d4,45393,1731962440107 in 1.8930 sec 2024-11-18T20:44:07,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-11-18T20:44:07,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=db394ce1aa4c72690b92156fd4dc5d2d, UNASSIGN in 1.8980 sec 2024-11-18T20:44:07,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-18T20:44:07,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9000 sec 2024-11-18T20:44:07,039 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731962647039"}]},"ts":"1731962647039"} 2024-11-18T20:44:07,040 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T20:44:07,040 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T20:44:07,041 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9090 sec 2024-11-18T20:44:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T20:44:07,278 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:44:07,278 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T20:44:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:44:07,280 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:44:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-18T20:44:07,282 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:44:07,286 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,289 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A, FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B, FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C, FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/recovered.edits] 2024-11-18T20:44:07,292 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6456eb2c85944c6dabf124a53031e1f9 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/6456eb2c85944c6dabf124a53031e1f9 2024-11-18T20:44:07,293 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/9efc740412ce41cc8dbc76e1ba7e6aa8 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/A/9efc740412ce41cc8dbc76e1ba7e6aa8 2024-11-18T20:44:07,295 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/36a11b6c6b554e0b9835e1089a0a9f4f to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/36a11b6c6b554e0b9835e1089a0a9f4f 2024-11-18T20:44:07,296 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/ac5c2c57ffe84bb2ab6b68cb5d4c93a7 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/B/ac5c2c57ffe84bb2ab6b68cb5d4c93a7 2024-11-18T20:44:07,297 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/11ac2c3def884f689590e04afd78395e to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/11ac2c3def884f689590e04afd78395e 2024-11-18T20:44:07,299 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/d46fa0a2aa404a28b4d5ad6a3f51a697 to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/C/d46fa0a2aa404a28b4d5ad6a3f51a697 2024-11-18T20:44:07,301 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/recovered.edits/342.seqid to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d/recovered.edits/342.seqid 2024-11-18T20:44:07,301 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/default/TestAcidGuarantees/db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,301 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T20:44:07,301 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T20:44:07,302 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-18T20:44:07,304 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111807cdfee7e46c49baa7a232d1dcd19f4e_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111807cdfee7e46c49baa7a232d1dcd19f4e_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,305 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111809bb56aafb094fceb70506cd205de139_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111809bb56aafb094fceb70506cd205de139_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,306 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180d310c786d9741fca444c8064d346b9a_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180d310c786d9741fca444c8064d346b9a_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,306 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118218bd752a1364005b9ff6a6917b4f274_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118218bd752a1364005b9ff6a6917b4f274_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,307 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118365c2518aeda47b5ae0d8bb7afd78360_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118365c2518aeda47b5ae0d8bb7afd78360_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,308 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111837b0d1d5bc1f4de8abf4238147c24e25_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111837b0d1d5bc1f4de8abf4238147c24e25_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,309 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184a63e53b38774f7ba84b43b8102efe2c_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184a63e53b38774f7ba84b43b8102efe2c_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,310 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186cdcc8e5d7a74d48bddc6965d10d1686_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186cdcc8e5d7a74d48bddc6965d10d1686_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,311 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118862a9a5348ce4745a9015ebb5f5eb9a5_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118862a9a5348ce4745a9015ebb5f5eb9a5_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,313 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188f01a03b3dd54944aedb72c46ba9a840_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188f01a03b3dd54944aedb72c46ba9a840_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,314 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b0d7a574c7e24d5e93b403aaaaec2d96_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b0d7a574c7e24d5e93b403aaaaec2d96_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,315 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bc1530d18c9847ef9d52d5dd50f88092_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bc1530d18c9847ef9d52d5dd50f88092_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,316 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c200f9a31e804109af18d0e0847c6ee1_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c200f9a31e804109af18d0e0847c6ee1_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,318 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118cc05e49cd62b4f2fb76aebfc088b6baf_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118cc05e49cd62b4f2fb76aebfc088b6baf_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,319 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ce3e285d6d7247739319ff53429ce07b_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ce3e285d6d7247739319ff53429ce07b_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,320 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d5211b8f894a420fa22bdc241e13b005_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d5211b8f894a420fa22bdc241e13b005_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,321 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118da6d8a460dcd499290405d25de89de45_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118da6d8a460dcd499290405d25de89de45_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,322 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ed5edd25b93c417fb9c7d9c431605f79_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ed5edd25b93c417fb9c7d9c431605f79_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,323 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f267428bbbc14ca2842bbf6e06d0684c_db394ce1aa4c72690b92156fd4dc5d2d to hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f267428bbbc14ca2842bbf6e06d0684c_db394ce1aa4c72690b92156fd4dc5d2d 2024-11-18T20:44:07,324 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T20:44:07,327 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:44:07,329 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T20:44:07,332 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T20:44:07,333 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:44:07,333 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T20:44:07,333 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731962647333"}]},"ts":"9223372036854775807"} 2024-11-18T20:44:07,335 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T20:44:07,335 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => db394ce1aa4c72690b92156fd4dc5d2d, NAME => 'TestAcidGuarantees,,1731962613400.db394ce1aa4c72690b92156fd4dc5d2d.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T20:44:07,335 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T20:44:07,335 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731962647335"}]},"ts":"9223372036854775807"} 2024-11-18T20:44:07,337 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T20:44:07,338 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:44:07,339 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-11-18T20:44:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-18T20:44:07,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T20:44:07,391 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:44:07,450 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 246) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=318 (was 322), ProcessCount=11 (was 11), AvailableMemoryMB=257 (was 389) 2024-11-18T20:44:07,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T20:44:07,450 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:44:07,450 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:44:07,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:44:07,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:44:07,451 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T20:44:07,451 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=370755489, stopped=false 2024-11-18T20:44:07,451 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:44:07,451 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c0a89b2656d4,46825,1731962439484 2024-11-18T20:44:07,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T20:44:07,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T20:44:07,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:44:07,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:44:07,452 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T20:44:07,453 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:44:07,453 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:44:07,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:44:07,453 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T20:44:07,454 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c0a89b2656d4,45393,1731962440107' ***** 2024-11-18T20:44:07,454 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T20:44:07,454 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T20:44:07,454 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T20:44:07,454 INFO [RS:0;c0a89b2656d4:45393 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T20:44:07,454 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T20:44:07,454 INFO [RS:0;c0a89b2656d4:45393 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T20:44:07,454 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(959): stopping server c0a89b2656d4,45393,1731962440107 2024-11-18T20:44:07,454 INFO [RS:0;c0a89b2656d4:45393 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T20:44:07,455 INFO [RS:0;c0a89b2656d4:45393 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c0a89b2656d4:45393. 2024-11-18T20:44:07,455 DEBUG [RS:0;c0a89b2656d4:45393 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:44:07,455 DEBUG [RS:0;c0a89b2656d4:45393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:44:07,455 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T20:44:07,455 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T20:44:07,455 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T20:44:07,455 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T20:44:07,456 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-18T20:44:07,456 DEBUG [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-18T20:44:07,456 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T20:44:07,456 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T20:44:07,456 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T20:44:07,456 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T20:44:07,456 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T20:44:07,456 DEBUG [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:44:07,456 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-18T20:44:07,468 INFO [regionserver/c0a89b2656d4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T20:44:07,479 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/.tmp/info/e07ec26ae2e643839115d2d60a2faf93 is 95, key is TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce./info:/1731962477201/DeleteFamily/seqid=0 2024-11-18T20:44:07,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742458_1634 (size=6514) 2024-11-18T20:44:07,656 DEBUG [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:44:07,857 DEBUG [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:44:07,883 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/.tmp/info/e07ec26ae2e643839115d2d60a2faf93 2024-11-18T20:44:07,907 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/.tmp/ns/a17bfee8935d4587a2af1857d337d515 is 93, key is TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce./ns:/1731962477161/DeleteFamily/seqid=0 2024-11-18T20:44:07,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742459_1635 (size=5906) 2024-11-18T20:44:08,057 DEBUG [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:44:08,222 INFO [regionserver/c0a89b2656d4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-18T20:44:08,223 INFO [regionserver/c0a89b2656d4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-18T20:44:08,257 DEBUG [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:44:08,310 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/.tmp/ns/a17bfee8935d4587a2af1857d337d515 2024-11-18T20:44:08,332 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/.tmp/rep_barrier/72cddb3919f24d3d982b771217db8c25 is 102, key is TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce./rep_barrier:/1731962477161/DeleteFamily/seqid=0 2024-11-18T20:44:08,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742460_1636 (size=6025) 2024-11-18T20:44:08,409 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:44:08,457 DEBUG [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:44:08,658 DEBUG [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:44:08,735 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/.tmp/rep_barrier/72cddb3919f24d3d982b771217db8c25 2024-11-18T20:44:08,755 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/.tmp/table/e5e950bc14654e6494d7f061c75d7846 is 96, key is TestAcidGuarantees,,1731962443729.4420199d334b1e279fd666dd4cb607ce./table:/1731962477161/DeleteFamily/seqid=0 2024-11-18T20:44:08,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742461_1637 (size=5936) 2024-11-18T20:44:08,858 DEBUG [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:44:09,058 DEBUG [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:44:09,158 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/.tmp/table/e5e950bc14654e6494d7f061c75d7846 2024-11-18T20:44:09,162 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/.tmp/info/e07ec26ae2e643839115d2d60a2faf93 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/info/e07ec26ae2e643839115d2d60a2faf93 2024-11-18T20:44:09,164 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/info/e07ec26ae2e643839115d2d60a2faf93, entries=12, sequenceid=90, filesize=6.4 K 2024-11-18T20:44:09,165 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/.tmp/ns/a17bfee8935d4587a2af1857d337d515 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/ns/a17bfee8935d4587a2af1857d337d515 2024-11-18T20:44:09,168 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/ns/a17bfee8935d4587a2af1857d337d515, entries=8, sequenceid=90, filesize=5.8 K 2024-11-18T20:44:09,168 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/.tmp/rep_barrier/72cddb3919f24d3d982b771217db8c25 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/rep_barrier/72cddb3919f24d3d982b771217db8c25 2024-11-18T20:44:09,170 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/rep_barrier/72cddb3919f24d3d982b771217db8c25, entries=6, sequenceid=90, filesize=5.9 K 2024-11-18T20:44:09,171 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/.tmp/table/e5e950bc14654e6494d7f061c75d7846 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/table/e5e950bc14654e6494d7f061c75d7846 2024-11-18T20:44:09,173 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/table/e5e950bc14654e6494d7f061c75d7846, entries=7, sequenceid=90, filesize=5.8 K 2024-11-18T20:44:09,174 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1718ms, sequenceid=90, compaction requested=false 2024-11-18T20:44:09,177 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-18T20:44:09,177 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T20:44:09,177 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T20:44:09,178 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731962647456Running coprocessor pre-close hooks at 1731962647456Disabling compacts and flushes for region at 1731962647456Disabling writes for close at 1731962647456Obtaining lock to block concurrent updates at 1731962647456Preparing flush snapshotting stores in 1588230740 at 1731962647456Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731962647458 (+2 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731962647458Flushing 1588230740/info: creating writer at 1731962647459 (+1 ms)Flushing 1588230740/info: appending metadata at 1731962647478 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731962647478Flushing 1588230740/ns: creating writer at 1731962647887 (+409 ms)Flushing 1588230740/ns: appending metadata at 1731962647907 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731962647907Flushing 1588230740/rep_barrier: creating writer at 1731962648314 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731962648331 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731962648331Flushing 1588230740/table: creating writer at 1731962648738 (+407 ms)Flushing 1588230740/table: appending metadata at 1731962648754 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731962648754Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39d17c8a: reopening flushed file at 1731962649161 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1898a0a2: reopening flushed file at 1731962649165 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a8624b3: reopening flushed file at 1731962649168 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@276682ac: reopening flushed file at 1731962649170 (+2 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1718ms, sequenceid=90, compaction requested=false at 1731962649174 (+4 ms)Writing region close event to WAL at 1731962649175 (+1 ms)Running coprocessor post-close hooks at 1731962649177 (+2 ms)Closed at 1731962649177 2024-11-18T20:44:09,178 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T20:44:09,258 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(976): stopping server c0a89b2656d4,45393,1731962440107; all regions closed. 2024-11-18T20:44:09,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741834_1010 (size=25344) 2024-11-18T20:44:09,268 DEBUG [RS:0;c0a89b2656d4:45393 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/oldWALs 2024-11-18T20:44:09,268 INFO [RS:0;c0a89b2656d4:45393 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c0a89b2656d4%2C45393%2C1731962440107.meta:.meta(num 1731962443376) 2024-11-18T20:44:09,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741832_1008 (size=18996825) 2024-11-18T20:44:09,273 DEBUG [RS:0;c0a89b2656d4:45393 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/oldWALs 2024-11-18T20:44:09,273 INFO [RS:0;c0a89b2656d4:45393 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c0a89b2656d4%2C45393%2C1731962440107:(num 1731962442369) 2024-11-18T20:44:09,273 DEBUG [RS:0;c0a89b2656d4:45393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:44:09,273 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T20:44:09,274 INFO [RS:0;c0a89b2656d4:45393 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T20:44:09,274 INFO [RS:0;c0a89b2656d4:45393 {}] hbase.ChoreService(370): Chore service for: regionserver/c0a89b2656d4:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-18T20:44:09,274 INFO [RS:0;c0a89b2656d4:45393 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T20:44:09,274 INFO [regionserver/c0a89b2656d4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T20:44:09,275 INFO [RS:0;c0a89b2656d4:45393 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45393 2024-11-18T20:44:09,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c0a89b2656d4,45393,1731962440107 2024-11-18T20:44:09,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T20:44:09,279 INFO [RS:0;c0a89b2656d4:45393 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T20:44:09,281 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c0a89b2656d4,45393,1731962440107] 2024-11-18T20:44:09,282 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c0a89b2656d4,45393,1731962440107 already deleted, retry=false 2024-11-18T20:44:09,282 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c0a89b2656d4,45393,1731962440107 expired; onlineServers=0 2024-11-18T20:44:09,282 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c0a89b2656d4,46825,1731962439484' ***** 2024-11-18T20:44:09,282 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T20:44:09,283 INFO [M:0;c0a89b2656d4:46825 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T20:44:09,283 INFO [M:0;c0a89b2656d4:46825 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T20:44:09,283 DEBUG [M:0;c0a89b2656d4:46825 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T20:44:09,283 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T20:44:09,283 DEBUG [M:0;c0a89b2656d4:46825 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T20:44:09,283 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster-HFileCleaner.large.0-1731962442101 {}] cleaner.HFileCleaner(306): Exit Thread[master/c0a89b2656d4:0:becomeActiveMaster-HFileCleaner.large.0-1731962442101,5,FailOnTimeoutGroup] 2024-11-18T20:44:09,283 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster-HFileCleaner.small.0-1731962442105 {}] cleaner.HFileCleaner(306): Exit Thread[master/c0a89b2656d4:0:becomeActiveMaster-HFileCleaner.small.0-1731962442105,5,FailOnTimeoutGroup] 2024-11-18T20:44:09,283 INFO [M:0;c0a89b2656d4:46825 {}] hbase.ChoreService(370): Chore service for: master/c0a89b2656d4:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T20:44:09,283 INFO [M:0;c0a89b2656d4:46825 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T20:44:09,283 DEBUG [M:0;c0a89b2656d4:46825 {}] master.HMaster(1795): Stopping service threads 2024-11-18T20:44:09,284 INFO [M:0;c0a89b2656d4:46825 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T20:44:09,284 INFO [M:0;c0a89b2656d4:46825 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T20:44:09,284 ERROR [M:0;c0a89b2656d4:46825 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (879580466) connection to localhost/127.0.0.1:46353 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:46353,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-18T20:44:09,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T20:44:09,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:44:09,285 INFO [M:0;c0a89b2656d4:46825 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T20:44:09,285 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T20:44:09,285 DEBUG [M:0;c0a89b2656d4:46825 {}] zookeeper.ZKUtil(347): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-18T20:44:09,285 WARN [M:0;c0a89b2656d4:46825 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T20:44:09,286 INFO [M:0;c0a89b2656d4:46825 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/.lastflushedseqids 2024-11-18T20:44:09,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742462_1638 (size=72) 2024-11-18T20:44:09,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T20:44:09,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1005490dfb30001, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T20:44:09,382 INFO [RS:0;c0a89b2656d4:45393 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T20:44:09,383 INFO [RS:0;c0a89b2656d4:45393 {}] regionserver.HRegionServer(1031): Exiting; stopping=c0a89b2656d4,45393,1731962440107; zookeeper connection closed. 2024-11-18T20:44:09,383 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7437fe26 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7437fe26 2024-11-18T20:44:09,384 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-18T20:44:09,696 INFO [M:0;c0a89b2656d4:46825 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T20:44:09,697 INFO [M:0;c0a89b2656d4:46825 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T20:44:09,697 DEBUG [M:0;c0a89b2656d4:46825 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T20:44:09,697 INFO [M:0;c0a89b2656d4:46825 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:44:09,697 DEBUG [M:0;c0a89b2656d4:46825 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:44:09,697 DEBUG [M:0;c0a89b2656d4:46825 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T20:44:09,697 DEBUG [M:0;c0a89b2656d4:46825 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:44:09,697 INFO [M:0;c0a89b2656d4:46825 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=768.37 KB heapSize=940.76 KB 2024-11-18T20:44:09,717 DEBUG [M:0;c0a89b2656d4:46825 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/93ddb351a6e5442d806142f20e5063dd is 82, key is hbase:meta,,1/info:regioninfo/1731962443457/Put/seqid=0 2024-11-18T20:44:09,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742463_1639 (size=5672) 2024-11-18T20:44:09,825 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T20:44:09,825 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-18T20:44:09,825 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T20:44:10,123 INFO [M:0;c0a89b2656d4:46825 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2124 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/93ddb351a6e5442d806142f20e5063dd 2024-11-18T20:44:10,151 DEBUG [M:0;c0a89b2656d4:46825 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cd4c7d1a62404d94a190ee1139ceb0e0 is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x5C/proc:d/1731962548961/Put/seqid=0 2024-11-18T20:44:10,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742464_1640 (size=45346) 2024-11-18T20:44:10,155 INFO [M:0;c0a89b2656d4:46825 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=767.81 KB at sequenceid=2124 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cd4c7d1a62404d94a190ee1139ceb0e0 2024-11-18T20:44:10,159 INFO [M:0;c0a89b2656d4:46825 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cd4c7d1a62404d94a190ee1139ceb0e0 2024-11-18T20:44:10,174 DEBUG [M:0;c0a89b2656d4:46825 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0b494eeaba3c4a308e55e90f662e72ad is 69, key is c0a89b2656d4,45393,1731962440107/rs:state/1731962442120/Put/seqid=0 2024-11-18T20:44:10,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073742465_1641 (size=5156) 2024-11-18T20:44:10,578 INFO [M:0;c0a89b2656d4:46825 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2124 (bloomFilter=true), to=hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0b494eeaba3c4a308e55e90f662e72ad 2024-11-18T20:44:10,582 DEBUG [M:0;c0a89b2656d4:46825 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/93ddb351a6e5442d806142f20e5063dd as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/93ddb351a6e5442d806142f20e5063dd 2024-11-18T20:44:10,585 INFO [M:0;c0a89b2656d4:46825 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/93ddb351a6e5442d806142f20e5063dd, entries=8, sequenceid=2124, filesize=5.5 K 2024-11-18T20:44:10,586 DEBUG [M:0;c0a89b2656d4:46825 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cd4c7d1a62404d94a190ee1139ceb0e0 as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cd4c7d1a62404d94a190ee1139ceb0e0 2024-11-18T20:44:10,588 INFO [M:0;c0a89b2656d4:46825 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cd4c7d1a62404d94a190ee1139ceb0e0 2024-11-18T20:44:10,589 INFO [M:0;c0a89b2656d4:46825 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cd4c7d1a62404d94a190ee1139ceb0e0, entries=170, sequenceid=2124, filesize=44.3 K 2024-11-18T20:44:10,589 DEBUG [M:0;c0a89b2656d4:46825 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0b494eeaba3c4a308e55e90f662e72ad as hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0b494eeaba3c4a308e55e90f662e72ad 2024-11-18T20:44:10,593 INFO [M:0;c0a89b2656d4:46825 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46353/user/jenkins/test-data/b4d61b5e-73fa-eaf7-fbe6-1ea09967c093/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0b494eeaba3c4a308e55e90f662e72ad, entries=1, sequenceid=2124, filesize=5.0 K 2024-11-18T20:44:10,593 INFO [M:0;c0a89b2656d4:46825 {}] regionserver.HRegion(3140): Finished flush of dataSize ~768.37 KB/786811, heapSize ~940.46 KB/963032, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 896ms, sequenceid=2124, compaction requested=false 2024-11-18T20:44:10,595 INFO [M:0;c0a89b2656d4:46825 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:44:10,595 DEBUG [M:0;c0a89b2656d4:46825 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731962649697Disabling compacts and flushes for region at 1731962649697Disabling writes for close at 1731962649697Obtaining lock to block concurrent updates at 1731962649697Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731962649697Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=786811, getHeapSize=963272, getOffHeapSize=0, getCellsCount=2319 at 1731962649698 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731962649698Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731962649698Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731962649717 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731962649717Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731962650128 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731962650151 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731962650151Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731962650159 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731962650174 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731962650174Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@272176c3: reopening flushed file at 1731962650581 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f0abe17: reopening flushed file at 1731962650585 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@176dbba5: reopening flushed file at 1731962650589 (+4 ms)Finished flush of dataSize ~768.37 KB/786811, heapSize ~940.46 KB/963032, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 896ms, sequenceid=2124, compaction requested=false at 1731962650593 (+4 ms)Writing region close event to WAL at 1731962650595 (+2 ms)Closed at 1731962650595 2024-11-18T20:44:10,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41353 is added to blk_1073741830_1006 (size=926070) 2024-11-18T20:44:10,598 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T20:44:10,598 INFO [M:0;c0a89b2656d4:46825 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T20:44:10,598 INFO [M:0;c0a89b2656d4:46825 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46825 2024-11-18T20:44:10,599 INFO [M:0;c0a89b2656d4:46825 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T20:44:10,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T20:44:10,700 INFO [M:0;c0a89b2656d4:46825 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T20:44:10,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46825-0x1005490dfb30000, quorum=127.0.0.1:56641, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T20:44:10,706 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@292d50c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T20:44:10,709 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T20:44:10,709 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T20:44:10,709 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T20:44:10,709 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/hadoop.log.dir/,STOPPED} 2024-11-18T20:44:10,712 WARN [BP-576781707-172.17.0.2-1731962436354 heartbeating to localhost/127.0.0.1:46353 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-18T20:44:10,712 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-18T20:44:10,712 WARN [BP-576781707-172.17.0.2-1731962436354 heartbeating to localhost/127.0.0.1:46353 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-576781707-172.17.0.2-1731962436354 (Datanode Uuid 6d06baad-4844-40b0-a3b2-9cc76010c5af) service to localhost/127.0.0.1:46353 2024-11-18T20:44:10,712 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-18T20:44:10,715 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/cluster_b02ea204-637b-6699-1ed5-681f735b812e/data/data1/current/BP-576781707-172.17.0.2-1731962436354 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T20:44:10,715 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/cluster_b02ea204-637b-6699-1ed5-681f735b812e/data/data2/current/BP-576781707-172.17.0.2-1731962436354 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T20:44:10,716 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-18T20:44:10,725 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T20:44:10,726 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T20:44:10,726 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T20:44:10,726 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T20:44:10,726 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0dfe058f-9665-d7b6-6eb3-1d45b30a250a/hadoop.log.dir/,STOPPED} 2024-11-18T20:44:10,742 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T20:44:10,893 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down